{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. What is PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Basic Operations of Torch Tensor\n",
    "\n",
    "- torch.empty\n",
    "\n",
    "- torch.rand & torch.rand_like\n",
    "\n",
    "- torch.zeros\n",
    "\n",
    "- torch.tensor\n",
    "\n",
    "- torch.ones & tensor.new_ones\n",
    "\n",
    "- tensor.size & tensor.view\n",
    "    \n",
    "- tensor.add & tensor.add_\n",
    "\n",
    "- tensor[:2] & tensor[:, 2]\n",
    "\n",
    "- tensor.item & tensor.tolist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000e+00, 3.6893e+19, 0.0000e+00],\n",
      "        [3.6893e+19, 5.6052e-45, 0.0000e+00],\n",
      "        [0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
      "        [0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
      "        [0.0000e+00, 3.6893e+19, 6.3906e-08]])\n"
     ]
    }
   ],
   "source": [
    "print(torch.empty(5, 3))  # 数字是0、无穷大或无穷小，也有一般值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.7725, 0.2715, 0.8958],\n",
      "        [0.6532, 0.1087, 0.6823],\n",
      "        [0.6785, 0.1455, 0.8845],\n",
      "        [0.0747, 0.5234, 0.7176],\n",
      "        [0.3726, 0.8948, 0.9992]])\n"
     ]
    }
   ],
   "source": [
    "print(torch.rand(5, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0, 0, 0],\n",
      "        [0, 0, 0],\n",
      "        [0, 0, 0],\n",
      "        [0, 0, 0],\n",
      "        [0, 0, 0]])\n"
     ]
    }
   ],
   "source": [
    "print(torch.zeros(5, 3, dtype=torch.long))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[5.5000, 3.0000],\n",
      "        [4.8000, 8.1000]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[5.5, 3], [4.8, 8.1]])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.4449, -1.7807,  0.5049],\n",
      "        [ 1.7135,  0.9671, -1.1484],\n",
      "        [-0.5042, -0.0579, -0.4078],\n",
      "        [-0.0944,  0.1655,  0.2645],\n",
      "        [-2.1989,  0.2521,  0.7114]])\n",
      "torch.Size([5, 3])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn_like(x, dtype=torch.float)\n",
    "print(x)\n",
    "print(x.size())  # torch.Size is in fact a tuple, so it supports all tuple operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 1., 1.],\n",
      "        [1., 1., 1.],\n",
      "        [1., 1., 1.],\n",
      "        [1., 1., 1.],\n",
      "        [1., 1., 1.]], dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "print(x.new_ones(5, 3, dtype=torch.double))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.9307, -1.2709,  1.0478],\n",
      "        [ 2.2759,  1.5239, -0.7382],\n",
      "        [ 0.2152,  0.5509,  0.1529],\n",
      "        [ 0.7290,  0.3528,  0.7645],\n",
      "        [-2.0706,  0.6540,  1.1559]])\n",
      "tensor([[-0.9307, -1.2709,  1.0478],\n",
      "        [ 2.2759,  1.5239, -0.7382],\n",
      "        [ 0.2152,  0.5509,  0.1529],\n",
      "        [ 0.7290,  0.3528,  0.7645],\n",
      "        [-2.0706,  0.6540,  1.1559]])\n",
      "tensor([[-0.9307, -1.2709,  1.0478],\n",
      "        [ 2.2759,  1.5239, -0.7382],\n",
      "        [ 0.2152,  0.5509,  0.1529],\n",
      "        [ 0.7290,  0.3528,  0.7645],\n",
      "        [-2.0706,  0.6540,  1.1559]])\n",
      "tensor([[-0.9307, -1.2709,  1.0478],\n",
      "        [ 2.2759,  1.5239, -0.7382],\n",
      "        [ 0.2152,  0.5509,  0.1529],\n",
      "        [ 0.7290,  0.3528,  0.7645],\n",
      "        [-2.0706,  0.6540,  1.1559]])\n",
      "tensor([[-0.9307, -1.2709,  1.0478],\n",
      "        [ 2.2759,  1.5239, -0.7382],\n",
      "        [ 0.2152,  0.5509,  0.1529],\n",
      "        [ 0.7290,  0.3528,  0.7645],\n",
      "        [-2.0706,  0.6540,  1.1559]])\n",
      "tensor([[-0.9307, -1.2709,  1.0478],\n",
      "        [ 2.2759,  1.5239, -0.7382]])\n",
      "tensor([-1.2709,  1.5239,  0.5509,  0.3528,  0.6540])\n"
     ]
    }
   ],
   "source": [
    "y = torch.rand(5, 3)\n",
    "print(x + y)\n",
    "print(torch.add(x, y))  # Same with +\n",
    "print(y.add(x))\n",
    "\n",
    "result = torch.empty(5, 3)\n",
    "torch.add(x, y, out=result)\n",
    "print(result)\n",
    "\n",
    "# Any operation that mutates a tensor in-place is post-fixed with an _. For example: x.copy_(y), x.t_(), will change x.\n",
    "y.add_(x)  # Inplace Addition   \n",
    "print(y)\n",
    "\n",
    "# Use standard NumPy-like indexing with all bells and whistles!\n",
    "print(y[:2])   # 前2个vector\n",
    "print(y[:, 1]) # 第1列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.8293, 0.5293, 0.6516, 0.2408, 0.2705, 0.9424, 0.7717, 0.2853, 0.9992,\n",
      "        0.2992, 0.0179, 0.5161])\n",
      "tensor([[0.7520, 0.8426, 0.7543, 0.1428, 0.0101, 0.8947],\n",
      "        [0.0070, 0.4272, 0.4334, 0.6758, 0.7053, 0.6084]])\n",
      "tensor([[0.7501, 0.3438, 0.0324, 0.7472, 0.7930, 0.7767],\n",
      "        [0.2618, 0.2395, 0.3214, 0.9655, 0.7449, 0.9825]])\n"
     ]
    }
   ],
   "source": [
    "# Resize shape: tensor.view\n",
    "print(torch.rand(4, 3).view(12))\n",
    "print(torch.rand(4, 3).view(2, 6))\n",
    "print(torch.rand(4, 3).view(-1, 6))  # the size -1 is inferred from other dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.12467533349990845\n"
     ]
    }
   ],
   "source": [
    "print(torch.randn(1).item())  # Get a Python number from a tensor containing a single value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.9341114163398743, -0.3458023965358734, -0.2878834903240204], [-2.503565549850464, -0.7918197512626648, -1.3584527969360352]]\n"
     ]
    }
   ],
   "source": [
    "print(torch.randn(2, 3).tolist()) # Get a Python list from a tensor containing many values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Torch Tensor < -- > NumPy Array\n",
    "\n",
    "They will share their underlying memory locations, and **changing one will change the other**.\n",
    "\n",
    "- tensor.numpy() --> numpy array\n",
    "\n",
    "- torch.from_numpy(numpy array) --> torch tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 1., 1., 1.],\n",
       "       [1., 1., 1., 1.],\n",
       "       [1., 1., 1., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.ones(3, 4)\n",
    "b = a.numpy()\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[2., 2., 2., 2.],\n",
      "        [2., 2., 2., 2.],\n",
      "        [2., 2., 2., 2.]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[2., 2., 2., 2.],\n",
       "       [2., 2., 2., 2.],\n",
       "       [2., 2., 2., 2.]], dtype=float32)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.add_(1)\n",
    "print(a)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2. 2. 2. 2. 2.]\n",
      "tensor([2., 2., 2., 2., 2.], dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "a = np.ones(5)\n",
    "b = torch.from_numpy(a)\n",
    "np.add(a, 1, out=a)\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Autograd: Automatic Differentiation\n",
    "\n",
    "Central to all neural networks in PyTorch is the ***autograd*** package. It provides automatic differentiation for all operations on Tensors. It's a define-by-run framework, which means that your backprop is defined by how your code is run, and that every single iteration can be different.\n",
    "\n",
    "## 2.1 Tensor\n",
    "\n",
    "***torch.Tensor*** is the central class of the package. If you set its attribute ***.requires_grad*** as True, it starts to track all operations on it. When you finish your computation you can call ***.backward()*** and have all the gradients computed automatically. The gradient for this tensor will be accumulated into ***.grad*** attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 1.],\n",
      "        [1., 1.]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "x = torch.ones(2, 2, requires_grad=True)  # Defaults to False  x.requires_grad_(False) will change to False in-place\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[3., 3.],\n",
      "        [3., 3.]], grad_fn=<AddBackward0>)\n",
      "<AddBackward0 object at 0x122257978>\n"
     ]
    }
   ],
   "source": [
    "y = x + 2\n",
    "print(y)\n",
    "print(y.grad_fn)  # y was created as a result of an operation, so it has a grad_fn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[27., 27.],\n",
      "        [27., 27.]], grad_fn=<MulBackward0>)\n",
      "tensor(27., grad_fn=<MeanBackward1>)\n"
     ]
    }
   ],
   "source": [
    "z = y * y * 3\n",
    "out = z.mean()\n",
    "print(z)\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Gradients\n",
    "\n",
    "Generally speaking, ***torch.autograd*** is an engine for computing vector-Jacobian product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "out.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[4.5000, 4.5000],\n",
      "        [4.5000, 4.5000]])\n"
     ]
    }
   ],
   "source": [
    "print(x.grad) # d(out)/dx "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Neural Networks\n",
    "\n",
    "Neural networks can be constructed using the ***torch.nn*** package.\n",
    "\n",
    "***nn*** depends on ***autograd*** to define models and differentiate them. An ***nn.Module*** contains layers, and a method ***forward(input)*** that returns the output.\n",
    "\n",
    "A typical training procedure for a neural network is as follows:\n",
    "\n",
    "- Define the neural network that has some learnable parameters (or weights)\n",
    "\n",
    "- Iterate over a dataset of inputs\n",
    "\n",
    "- Process input through the network\n",
    "\n",
    "- Compute the loss (how far is the output from being correct)\n",
    "\n",
    "- Propagate gradients back into the network’s parameters\n",
    "\n",
    "- Update the weights of the network, typically using a simple update rule: ***weight = weight - learning_rate * gradient***\n",
    "\n",
    "## 3.1 Define the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We just have to define the ***\\_\\_init\\_\\_*** and the ***forward*** function, and the ***backward*** function (where gradients are computed) is automatically defined for you using ***autograd***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):  # nn.Module: Base class for all neural network modules. Your models should also subclass this class.\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5, stride=1, padding=0)  # 为什么输入需要是32*32 ？\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(in_features=16 * 5 * 5, out_features=120)  # An affine operation: y = Wx + b\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "    \n",
    "    # Conv2d和Linear属于Layer范畴，来自于nn，定义在__init__里，而max_pool2d和relu属于functional范畴，来自于F=nn.functional，定义在forward里！\n",
    "    # Structure: input-->conv1-->relu-->max_pool2d-->conv2-->relu-->max_pool2d-->view-->fc1-->relu-->fc2-->relu-->fc3\n",
    "    def forward(self, x):\n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))   # Max Pooling over a (2, 2) window\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), 2)        # If the size is a square, you can only specify a single number\n",
    "        x = x.view(-1, np.prod(x.size()[1:]))             # batch dimension之外的dimension相乘\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
      "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
      "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = Net()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[[[ 0.0096, -0.0146,  0.1165,  0.1517,  0.1280],\n",
       "          [-0.0387, -0.1381, -0.1021, -0.0893,  0.1611],\n",
       "          [-0.1267,  0.0353,  0.1368, -0.1255,  0.1639],\n",
       "          [ 0.0874, -0.0690, -0.1079,  0.1732, -0.0358],\n",
       "          [-0.1358,  0.0208,  0.1711,  0.1663, -0.1493]]],\n",
       "\n",
       "\n",
       "        [[[-0.1793,  0.0615,  0.0538, -0.1476,  0.0203],\n",
       "          [ 0.0992,  0.1008,  0.1627, -0.0625, -0.1988],\n",
       "          [ 0.1879, -0.1236, -0.1700, -0.0055,  0.0679],\n",
       "          [-0.1129, -0.1276,  0.0227, -0.0375,  0.1788],\n",
       "          [ 0.1206, -0.0703, -0.1270, -0.0672, -0.0485]]],\n",
       "\n",
       "\n",
       "        [[[ 0.1248,  0.0423,  0.1732,  0.0402, -0.0659],\n",
       "          [-0.1116,  0.0453,  0.0711,  0.1744, -0.0561],\n",
       "          [ 0.0306, -0.0745,  0.0090, -0.0544,  0.1572],\n",
       "          [-0.1756, -0.1798, -0.0988,  0.1113, -0.1086],\n",
       "          [-0.1931,  0.0937, -0.1316,  0.0583, -0.0173]]],\n",
       "\n",
       "\n",
       "        [[[-0.0929, -0.1267,  0.0503,  0.0175,  0.1571],\n",
       "          [ 0.0360, -0.0737,  0.1474, -0.1901,  0.0339],\n",
       "          [ 0.1157,  0.1043,  0.1153,  0.0789,  0.1536],\n",
       "          [-0.0443,  0.1551, -0.0928, -0.1692, -0.1145],\n",
       "          [-0.0602,  0.1319,  0.1354,  0.1954,  0.1976]]],\n",
       "\n",
       "\n",
       "        [[[ 0.0590,  0.1275, -0.0317, -0.0723, -0.1197],\n",
       "          [-0.0434, -0.0190, -0.1204,  0.0958,  0.0584],\n",
       "          [-0.0280, -0.0725,  0.1066,  0.0999,  0.1252],\n",
       "          [ 0.1219, -0.1818, -0.0244, -0.0916,  0.0664],\n",
       "          [ 0.1877, -0.1215,  0.0192,  0.1504,  0.1419]]],\n",
       "\n",
       "\n",
       "        [[[ 0.1578, -0.1095,  0.1229,  0.0533,  0.0833],\n",
       "          [-0.0308, -0.0675, -0.0763, -0.0532,  0.0209],\n",
       "          [-0.1452,  0.0308, -0.1144, -0.0716,  0.1649],\n",
       "          [-0.0617, -0.0484,  0.0090,  0.1748,  0.0834],\n",
       "          [ 0.1366,  0.0070, -0.0380, -0.1730, -0.0234]]]], requires_grad=True)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(net.parameters())[0]  # conv1's .weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 0.6037,  1.4687, -1.9032,  ...,  0.4374,  0.6948,  0.9238],\n",
      "          [-0.8192,  0.4986, -0.2325,  ..., -0.4423, -0.2899, -1.5858],\n",
      "          [-1.5947, -0.6475,  1.4343,  ..., -0.5507,  0.3688, -1.0173],\n",
      "          ...,\n",
      "          [ 0.6210,  0.9085,  0.0089,  ..., -0.6819,  2.6848, -0.0111],\n",
      "          [ 0.3846, -0.0431,  1.3146,  ..., -0.1887,  1.1367, -0.4960],\n",
      "          [ 1.0715, -1.3381,  1.5364,  ...,  0.0486,  1.4302, -1.7637]]]])\n"
     ]
    }
   ],
   "source": [
    "input = torch.randn(1, 1, 32, 32)\n",
    "print(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.0487, -0.0073,  0.0879, -0.0596, -0.0364, -0.0173, -0.0623,  0.0081,\n",
      "          0.0340,  0.0165]], grad_fn=<AddmmBackward>)\n",
      "tensor([[-0.0487, -0.0073,  0.0879, -0.0596, -0.0364, -0.0173, -0.0623,  0.0081,\n",
      "          0.0340,  0.0165]], grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(net(input))          # PyTorch中Model可直接作用于input，相当于对其进行前向传播计算，就像函数一样！实质是调用了Model的forward方法\n",
    "print(net.forward(input))  # Same with net(input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Loss Function\n",
    "\n",
    "A loss function takes the (output, target) pair of inputs, and computes a value that estimates how far away the output is from the target."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.1216, grad_fn=<MseLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "output = net(input)\n",
    "target = torch.randn(10).view(1, -1)\n",
    "loss = nn.MSELoss()(output, target)\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, when we call ***loss.backward()***, the whole graph is differentiated w.r.t. the loss, and all Tensors in the graph that has ***requires_grad=True*** will have their ***.grad*** Tensor accumulated with the gradient."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<MseLossBackward object at 0x1242f7be0>\n",
      "<AddmmBackward object at 0x1242f7c18>\n",
      "<AccumulateGrad object at 0x1242f7be0>\n"
     ]
    }
   ],
   "source": [
    "print(loss.grad_fn)\n",
    "print(loss.grad_fn.next_functions[0][0])  # Linear\n",
    "print(loss.grad_fn.next_functions[0][0].next_functions[0][0])  # Relu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Backprop\n",
    "\n",
    "To **backpropagate the error** all we have to do is to ***loss.backward()***. You need to **clear the existing gradients though**, else gradients will be accumulated to existing gradients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1.bias.grad before backward: \n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "conv1.bias.grad after backward: \n",
      "tensor([ 0.0026,  0.0107,  0.0082,  0.0096, -0.0055, -0.0157])\n"
     ]
    }
   ],
   "source": [
    "net.zero_grad()           # zeroes the gradient buffers of all parameters\n",
    "print('conv1.bias.grad before backward: ')\n",
    "print(net.conv1.bias.grad)\n",
    "loss.backward()\n",
    "print('conv1.bias.grad after backward: ')\n",
    "print(net.conv1.bias.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 Update the Weights\n",
    "\n",
    "The simplest update rule used in practice is the **Stochastic Gradient Descent (SGD)**:  weight = weight - learning_rate * gradient\n",
    "\n",
    "We can implement this using simple python code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "for f in net.parameters():\n",
    "    f.data.sub_(f.grad.data * learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Package ***torch.optim*** implements various different update rules: **SGD, Nesterov-SGD, Adam, RMSProp, etc**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01)\n",
    "\n",
    "# Belows are in your training loop\n",
    "optimizer.zero_grad()                  # zero the gradient buffers\n",
    "output = net(input)\n",
    "loss = nn.MSELoss()(output, target)\n",
    "loss.backward()\n",
    "optimizer.step()                       # Does the update"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Training a Classifier\n",
    "\n",
    "What about data?\n",
    "\n",
    "Generally, when dealing with image, text, audio or video data, you can load data into a *NumPy Array*, then convert it into a **torch.\\*Tensor**\n",
    "\n",
    "**data** --> **Numpy Array** --> **torch.\\*Tensor**\n",
    "\n",
    "Steps to train an image classifier:\n",
    "\n",
    "- Load and normalizing the CIFAR10 training and test datasets using *torchvision*\n",
    "\n",
    "- Define a Convolutional Neural Network\n",
    "\n",
    "- Define a loss function\n",
    "\n",
    "- Train the network on the training data\n",
    "\n",
    "- Test the network on the test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Loading and Normalizing CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(), \n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)  # 下载数据并进行Transformation\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True, num_workers=2)          # 数据加载器，可按batch_size进行mini-batch式的遍历\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=4, shuffle=True, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5\n",
    "    nping = img.numpy()\n",
    "    plt.imshow(np.transpose(nping, (1, 2, 0)))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJztvXmQXtd1H/i779u33jd0Y2lsBMF9E0VRCiVRckLZjuXyJB7LHkeZqMLMVFITT6Uqkcd/JKqZqSQ1qcRJle1YFduSZzTWYjs2o3iRhpFMJdq4SSABEnsD3eh9/fb13fnjnPvO6Q0AAQqNbt9fFdkf7nvfe3d77zvn/M5irLXw8PDw8Nj9CHa6Ax4eHh4e7w78C93Dw8Njj8C/0D08PDz2CPwL3cPDw2OPwL/QPTw8PPYI/Avdw8PDY4/Av9A9PDw89ghu64VujHnOGHPWGHPBGPPpd6tTHh4eHh7vHOZWA4uMMTEA5wD8GIApAC8D+IS19sy71z0PDw8Pj5tF/Da++ySAC9baSwBgjPkigI8D2PaFns1mbU9Pz23c0sPDw+MvH2ZmZhattYM3Ou92XuhjACbVv6cAvPd6X+jp6cHzzz9/G7f08PDw+MuHz3zmM1du5rzbsaGbLdo22W+MMc8bY14xxrxSrVZv43YeHh4eHtfD7bzQpwAcUP/eD2B640nW2s9aa5+w1j6RzWZv43YeHh4eHtfD7bzQXwZw3Bhz2BiTBPBzAF54d7rl4eHh4fFOccs2dGtt2xjzDwD8OYAYgN+21p5+p9d5+qf/GV+vFbXFEvw3CKUtRn8TgVn3FwBiPIpEStqSMfocj8m9uAkhW4biMbEQJfmeCd3WaQMArk3PRW2n3nwLADA/cw0A8N4nH42OjR8+DAAIkpmorWWpA2EofQv4/tZQm4mr31Vu67SlHwGP1cp04Df/1f8OjbXy69HndIHumc0LAX1tbhEAUGvUorZ8Lsd9o3vNLsg4wzjdMwykHzxFGMh0R21JQ2M13MdCVyE6FueFaaMpHY11AAC5fDpqqtZLAICRRD8AoDG9Fh27fI7mO2jK/jCW5qtU60Rt9QYdN7Ekz0E+OrZSrtMcrMrYSzU6/7//+b+NjZiYJnOldgBrNmkMyWRKhsIbtdkiU2KtsSDnN9i82JJHzM1Hp9GI2hIJ6i/cfg7UXuBNH8TlGh1L3zVG7w8+j787NDYq1wjou8vLlagpn+8DAAzvE46tWFoFAPR0DQMA7jv5QHSs1aHvfvnLvxu1/fhzP0P96Ug/5s9dhsaDf/1no89uLjvaUms2/KWBrT9kYuqQWfd33Vet7AU5tvl859WnbcOWD1vVEbPB+2/d+fwg2i1O0Ne4ngOhNXSNdijrHQtpDOf+85e2/+INcDukKKy1fwLgT27nGh4eHh4e7w5u64X+rnQg6oF0JZZwErT8OsedxM0/aAn1qx7jfyQS0ubOi+nznBDEUmJCSe8Jvn1GNa5MksT6xf/3D6K2Ny/PAgBWWep7+Y23o2Mffs9JAMAHP/j+qK37wDEAQDOUzqVYIjbcIauEskha0CvjpJvO9j/5bVuPPq8WSZpcLoqkW241N92r2GBpmbmN7j6RatfW6LuNWjtqiwUkVaczIl0/fozGd3Wa5uXCxER07N6H7gMANJW2Ued7VlZFYnRSUypJ121ByPPRffvpGuUVGd9amc5HMmoL+LshS6T1pvS7UiOptqGkfIRK3dmAep0keaOkrZDPbyllI2QNslqjsVglNcM66Vok+rDj1l3a0mnSkuotltrVfm00eM3UTU3cSZ1yXiLh2uiezbpoANUGz1VatLV7H3gEAJBJy55sXmLNc47WcaCvLzp2eeIc9TUlmmezSfvtyoQ4uuWhHiiIBqqR0NKy08C3kLiddGuCdbKx+j8h3NJq7M7j+Vab3l3Xqq6FTmpXjSH3RM7fLL3r4bl1SSiNInBSeKu17j4AEOPPcWVCiJnbD9z3of8eHh4eewT+he7h4eGxR7DjJpfoF8WICszaC2KK+HQmlMiUorkjEzESEWy4Xu0CgIh/iNG91OURMNGhNbzLk0QkXl0UVb2ZJU/NbD+psLNFIcK+8MJ3AACn3zwbtf31/47Io/sffY/qL3fEmVe0GrrhLyCqWmC2N7kk0rKUM2z+aIVCFIU8cVaTbnzdap3U8mQg1wg6NB/9XaJ6Hz1wD/3tH4naHjtIJpFzpy8CAOavyXx0DRBxPLxvOGorZMisMzslHq779xGJlw7I9BPLS797C9RWWhUzxWp5gsYUirqaKxAZyxYdrC0uRccqTJi2OnJda7c3uTTqZOKIKZOfM7k0lSknHqf5Gxqg+VhelTEZQ3OZTAhJvLK0TG3qusUKmS463LdmR/aaDbi/aqMGHfquUW0d3tgxNm01ZmUN9o2NAQASKTGnHThA5H2nLWa6K/y8LM7S3nlZmescbzvQPxC1nT1LpsaV5dWo7fjQQWjklZOC4X0XbmHq0s+hs4VEj4Q2a2xhwgmZDNVX7YTrTTNGPU02+p56L/DfttoT7nAYnSbnx/kl0a4rM+cy7be5KxNyHn/n6IkTdC3VyU6V5nd57lrUVqzdfpyOl9A9PDw89gh2XEJ3nIBVv7EJbowrdz5HaDrOUnv6ObdFfQ33c6jbnJTg3LvqNSHmkmmSAFttkcCmVkhSG3/4maht9m0igToJknjiIyJ9xnIkzZ6eFBfC0ufIBekfD49FbaPjJMm03M+/9m/ica5zzTJRI7aDJoeqNbpesVZW4yMCLKGY4xgz0pUmSQadQEjG8TGS4ob2Sb8fuOd+AEC2KXN67SpJGD94+Q0AQGaoV/rNZFSnJZJxNkMSa39BpL2+LH0OmnSeXttkkiT0MN8ftXX3kWRUXxYp0nWp7VzVEoqMNDQ+a2VtcT0JvUnzF4trjci50ylpmYnP/kHSMBZXRDJ22kBvv8zHzCy7jtYV0cy3aLfpfKOeSMNkZ1y5LYaOuNND4f3jJMCEukazThJ/Oi2TOnXlKvV3XjSKuekZAECOCe9yuSh9ZI22WFRaAWtw5ZJIlRsl9OnT340+Bykif/sOHpfrsuhv1FrEWPpN8CbQkrTdIjg9voVIaoP1522VgHArUrStnqGYe2/wMxoobaZapLn5zjdeitrarCm88f3vRW3O2aBcpn365Ac/Eh1bY83+ytvyrlhcoffR4f1HNw/qJuEldA8PD489Av9C9/Dw8Ngj2HGTiyM31/losh6qic9gA10Yap9s5xcaU8Qqf1n/YjkTzlqZIhO/8v98ITr2U3/1owCA4QExBSyXyeTSMGKmcGRaNkk6WysUH2HbMwQAGOz/UNRWmvwBAGBpSfyoDx0h1bTjtDpshrau2OuQoQ4dFXGWz1EEYKkm/sgFNnX0dIkJwKnyq6tE1sVC2Q6ZeBcAoFIUlfrSefJHvv/goajte6+fAgAsLJNKemBQfJVtiwZRKpaitmyM1NCBbolSrK3Wud9kwsjkZL7bHY427Yg5yMRIfY8lZXyNFs1ijX22y4qwqrko0kDIyGSw/ZzmemgPOD9wQGIievvEnzuZoLFeukpEYqUhi+YiAK9OzkRtrQ77Nqugxrb7zOabuPJFdtxpsI6c5Qhe3WE2bbUjU5j0Y5VJy6V5MaEszZDpZ3hY9nqtTOp+i6/R7igTA3/WUY1ufzbVHG1EZe6q9DtD+25o/N6oLeBI7KTa3ykea5Jv1VDPedO6uA3Zp86kus7IEjmzb/bZdzNk0dn0haSV/dTmcbU4qrepYhguXTgPAOhR5rR8Dz1fK3Oy3kfvpbGGHMuRMDJX+w+SKbO+fF/UdpLfM1NXJGL7ncJL6B4eHh57BDsuoTsvukD9tgT8k6qFqMBFV7IrXqB+dt1no37N69UKX0N+iZM9RGSuzJOL0RunzkXHfuyvfAAAUC6JZJJJU84SuywucCcPETmXZHIxlpR+XJwn8iNUUXknP/BhAEBhcF/U5pSRmOu39iTccI5GGF5PUpdjtSqRobYjfTs4QlrBshpLg6WUbILGWVkUKa60QNfoHRUpzrZJvlmYFTJt4iq5Kz70MBGm6S4Z++LUFABg/KC4PqZ6qE9rq0LYtiokBQ0wOWpisi1dpGqxKOe3HcmqmK1SmTSJpTVa9xVF1jmiO1RRfOmMSPwbceQeymMyMSG5SRyBXOgVzaK/nzSyeJokXpMSIrbN+V1KRXHrS7AWkVSagiPdElECI+U6F2OpUOU0anecSiuXcER3GDkCCGPaYqLZNkX6HNxH/TxxaH/U1qiSFuWikaemZOydDn3XGokQbrHUbjZEh2oksuIqOXyEpNWEFiG5T3pbN1gLsG2WxteJ3nSvtnJxRmzz/SOXQ+e+qIlVvl5Kqf8xXo+1ZdGiK0x8NjhCOZaUsQ8NkmR+bUqVg2jQfIwOD0VN+4bJYWL8OBHBr/zFi3JsPz2PB9R7oeLeaV5C9/Dw8PDwL3QPDw+PPYIdN7k46Gguw6pjTPkNx5tkKkhYToSkogRrTFxUSmIyWFggoq9eFRNK/xCp/hcnyRQQKH/gKpspzi1Lm+kQMWhal6K2wV5Si4plOn9kQAjCcVajlqpCfgxxptm20itFA3S6IeQYEzQ6qsyGm9s2IpFUxBJ/PjwmfsE9WSJtJq9KJasik4VHx8jvdXBEzCXDQ6QK3vOIpAeurs4DAN76zl/I+HqJ5CywqaVYVdGYSSIvj/aJ2caZFi7Ni1rpTBCVOn23pfyIi0zWuSRTgBBaem2XF8ncNV+ktlpD7Z3A+abLBObzQt5uRO8Aq80JMcuUKtQPk5K2XDep3vewH/rSqkTQVou0X9cW5qO2eR5zV16iR2PseJ5PZ3lsKk1wwOamqvjb2zip/qmUFIsZ6CczYCZDbZPKXDLN6x2PKfKSzQNzVyeipm7u0+AgjX15Sfrt8r6ZuJgdavWQ+7H9POpo1mSavmvUM+1SAK9PWxuub1Nppw2bktIpFdHsEtxt5Wu+ZRv9vXD6zahthqOWn3jmr0Rti5P0zE9fpIjYdJ+sbTpOe/jiaSmfPDBCz4tVppxqidZtbIyOfUtVbPvWN/4LAOBjH/lg1HZ2XsyhtwovoXt4eHjsEdxQQjfG/DaAnwQwb619gNv6AHwJwDiACQA/a61d2e4a14PLrKp/WTqRb5FIFa0yRZ2lGiQ5mFAkJdsgt7G4kg57LBEyy02RKipzJGUla+RaNJIWwurll0jqDLolkm3oAKXDNbHlqK3A0tsMKwPnJ+UaJ4+RRJ/JSD8yzHU1GxKV2mIJ0+WZ6WhpnInMUOe5ZWnleulzkyoycpSjO/tVzhWXbrinW6TwHF83kWRXMYi7YJ3To1Yr0u8LZ4kALa2KxPjwySMAgNlZkqAXrglRtG+MSKHBPnHvChMktXeaIl1PLdLWqbL75JFDovXU2P2wUpe94FzIasplrsTSj3NnDRTx6EgvxET6Hdwn87AR2SzNZdUq6TND12io1LTL3N9UjvZasS4SWI1J32Zd1mxkjPIAjY1J5cbVZdpILiVrb1bu2VWguTp/VTTEJhfw6FLk86FD4wCAo8dI0xpfkIjVU6fIrbSmJO7yLK0jQpEIAyYwTR/tzb4eiYBORK6MMvZ4grSBfGH7eYwlRYvI8RxpZ4YOS/A6sNNF4rpcTO11e576sbIiroHZHN2jq1uKrrRD7ZIoGhoATHFEbKksrrRXLl4AACST8sw1iyS1r7G2vbQ6ER2bnSQNSLtDFtdoLns5whoALn2TSNDxw7Te6WxO9ZvW+ey8vFuS7FJcm72lVymAm5PQPwfguQ1tnwbworX2OIAX+d8eHh4eHjuIG0ro1tqXjDHjG5o/DuBD/PnzAL4J4J/cSgci07Ly3XP25nZM2ezYjSi5SpkMu1VuilyBj+VEWpiep1/pRqAKS7Ad+WA/SZjpsgz/9YtkayzcL7+wNZaqFxdESg0yJHnV1uj6VZXz4lyFPo8eFck4VSD7aqct92o6dyrnVqj9tthQ3tG2Q5eV8Tpei9WKSKsrLB3WVHGKdIrmoadH5shleWyxxFtUpdFclr5KUUsyNEdjKjjJ2YPnLM1pXJUSPLyf5qGurtvhbuZVkYxXLtN1lzj3RToj7m51lohryv7opmGlKOtSb7oSdDTPcSU+JV1bXtoGriOhOzPoyJDY/sslmofLyibek+egJ87A2FRBJX1chrCoJO6hYbJ1h0o7WWE30iTbhY8fHY+ONTjwJszJfMT5kV1T67LCfM4Kz1GjJeue4T1/7ZJI+fUV+m5VZUNscwBSH5clPDQmOXw6XMpwit1QASDP6+7c7wBg7cp5aGSUROo4Mqv2upPGQ51XJ3oknOou55/5AeU9+bP/+J+kb6zi/91f+gdRW/9+euZirshIQvba9BRp86mkaPh/63/6FADgt/7dr0ZtIT/7iV7aA6f/m+RoKVZorg4ePRa1DY6SVnngyHjUds/DDwIAJpZpvgsH5Fj3IdKmOh0dmEVjqeHWcas29GFr7QwA8N+hG5zv4eHh4fEjxo+cFDXGPG+MecUY80q1evv5fj08PDw8tsatui3OGWP2WWtnjDH7AMxvd6K19rMAPgsAo6Ojm4wGUc2+ddUpWPVQdTi7+ynissuSehlvvhEda3M195Z2Z2K3xlBxtTGOiMxYiuwbyoopIB8j1fjapKiNE0ukQk9eFZKpuECk3wefIrNDFuKC1qi5tKuiDoct6luzJQSR7bjcGy63x+ax6/wtYj24Tvrcjvw2u9qPyaTMX5kjZ7vyogYfPsQ5ZdiVLKWj4bpJbT16SHJvXD1NdSe7lItknaNHV9kEcPyYkMrdfK+paUniX+fIxaV52TKO0FpaIlX2e6/+IDoWReSqiMBcF815RZkWnPXKuTTq4iguY/D+Q2JCyffKemzEmTdpb9VUdGWD3SdTap+mQPMV47qhaZXzp8CyUkGtQXuW3BatqrG6n02NLm1tr3J7++FVIuayGVkz8Jgnlxejpots5sp205iuXRa3xR++8hoAYHZK1qCfCd6myt3TqTEJPkums0SXmEEGR8n8Mvj4k1HbELvpZZQL5rc3mFy6ekVxb3G/NSkai7lcK6rmp6v76z4oa8x9Dz4MALj/oUeiti9yPqav/cmfRm1/53/+e3QtdiyYmRZT2FqJ9tjFH74WtT1ykswfI6qAx9f+nMw7J56kvXn4YbnnSTal9I9IlKfh/RnWxJzmrEVXzhHpalRe47EjzuQiYw+v83zfLG5VQn8BwCf58ycB/PFt98TDw8PD47ZwM26LvwciQAeMMVMA/imAfwHgy8aYTwG4CuBv3moHXG4Wq7IWhpxt0SalrZNhcrGPpIVMSyT0QkjugnVVHT0TJ7JptEtIy0SSy31dI2lydU3cm0qcs8HEVTBJk7SBdk2kw54ukrg+9CBJwYf6RYSoN0jKKhbEhWpqjSSeoCr96FSJZDIgsstauX7bRQ8pCT1wgTHX+QFX1dVQKPB8qIyDmTzdK1QuXQHfK5WjMcfSMvYqaxYtRWgO9BCp1yiLq9XpC0RoljmQp1/N35m3icCeWRIJyZGbIcTN0hU6KNdo/eoNIVYTTF6l0yqPCOga9bYaC+daafEecCQwABw/wRLmYZF015jYSouAGcFl09PSpOHx3TMkuVwSPJa5IkneHeW2mOBAoYSa7zzv68F+kVxDfgRT7HY6f0ak3PoK9XH0hMpQGKfrTam1XZoh7eEUj/3KBSmBOH+N8+7ovDfsnlpvyN49VKA+9VkKvluYEs02/yDN/eixe6K2t1nqrDWEKN2IeFq0IEd8rsuSyvKkDkCKcXInV9RFJXiM1jiIy/Py7I9R0Yjf+fV/H7XNMnn76muvAgCmJsWVdpFz61x9862o7c//7OsAgIHDUlji479I0nfvCDs4qHuiszkgyrLmmWnJ81JkDWFxifqz/4iso+F9nVTEdOd6kYM3iZvxcvnENoc+sk27h4eHh8cOwEeKenh4eOwR7Hgul6RhEks0ZKRTnOw/oarWc8rUdg+n/KwIgRFrksqbTSvCilNsdltRTcslUiPfvEIq/VJdouHCJEd6xUVlCtmUU05I29QcqWy/+wUi9Z7/20KWTFzkYhZWCL/DJ6ni99g+UfcLSTK/nHmbzBVhQvyMDx6hyMt2a7P6dT0/9LiawJ4uMvm0FAFaaZA5IK7UutUVItaaazQfbUXIZUDX6KjcNh1WKxdUsY4i12UNuRDFmfMXo2OlNSKTbSDXKHCEnPZHnpvj+eKITqMLOvBpujiF8+PvKDOCqxfrWrq7ZOzj46Q2J3rknlNzQnRvhKtp21bRqV0Z9idXpNeFa5TLo8zhwDauCVP6brdaxoBZslUJLkabidREi/q7osxIYwcpwvDhhx6K2hpshnnz1A+jthWO3J1kUrm0LCaxOOc/0dp8q+3qo8qe6U2TyaUQJ5NSS813a4H2/7VJSZs8ySRroExsYkQjxFQaZOeHvi7kgs0wcZWC10XMtl2dVGVmTDrzlappOzBEz/DT739f1PafX/gjAMAqmzvrKrp3+OA4AODJv/fhqK3QxyZSRc4a50PPE5ctSE3bJO81bSJxPvXxhJzXw4efPUzOB/oZzbAJMZNURVd4333zNXEKeKfwErqHh4fHHsGOS+iOa1iaF+KsWSNJIxHTboj0i11IMllnuqJjqRxJeHHlUhbn36q4ihQN2J2v3CQJuRqXwguD++lepiHS5/w0ESeVZSHpggxd99wESWAvfkv6MXGVpPwzV6X01r3HSXs4OSYS4yMPk2tfq87STUa0DWOIwDNW+h0VtriOiD7WL5JBmt2j5ptC0q1WqG9BS8Yyx8dTWTo/nZWxtDhPSq0ukY6lVc54qTIVZliSm7hErnJhSxdXoLnq7Rbm0UnVgXKRzKWYfG47KW5zZsqOYn0DLlRhFJnstIcCS9IHD0ik49IiXf/EyAlp66zP96Gxb4SkvmpJMjw2uMCGluybXJijd5TItJyeF5bOiso9s7hMeyGv12qYoxqztN7Dijg7fC/lEtKk+WXOFJrvkb073k+fm1zUI64yGq6t0jp21LpIWTeZ5+UiSbMTAa1xqIjp5mWSzA8eE432g89QQZjFZcnrc+m7quAD1mc7dMU3jMpR5AjSUBXwcKvSadN5unxFmKB/qbotmGFtJDUgRPOxYVr7FO+7nHKtNFy+rlGTtY2xS2+xKm3gbJb5IdLu3vr+y9Ghl77xortY1JbgoT7+1NNR28c+8TMAgDg7OPQrYj/DWTtTypUxzeP7Jm4dXkL38PDw2CPwL3QPDw+PPYIdN7ksc8Tbq9//TtRW5PSYmYyoSsMpjrxj9ewCxP/12adJVRmMK9MIq51BQpS2BkdrnrtCJp1r6hpHD5G6PNAtpFcuR5+XlsR0EeeiDc99aJzO75FjL58ic02lIxFkb18itfzc6xK9N3eNVO/xo0RAjR4R0ivkRE9hR0wu1lU7v15yLmUeSDIhODsjJFaLf7oTiuyqdWi+UpywH7pWI39cnBOTQYPNMNkh8bM/+zb5I1crrO4rf10hzERuaPD4siqqN88JrEqRr76qL8tErfNPBlRSJ5V3uM3JuYYPkor86CMPR8e+/wqRTJOXZqO2qQki9foH78dGFDhtbUr5sq9woqyVoqrdGrDvMdcs7VemkUSV1r1dE2I1y8eHesVckttPJH/yMPl4H1SRts0SmQIunz4dtV1ZoL2TGBSzjYu4bLZorUIjc+uKXiRVsOncUonPk77NVmmvVDhZVLop870vQ20JlRq5q59iKUZHxAwj6b8IRvngN3mtjAr9DNj80m4rcjFOnwN+zjuqjmmZTVwVlfrWEenDx8VH3uX66nCkb60m6a5cnEKnJX0b4GjegtqTNWdO4bnUtU1rvC7TM8pMzAT6aVU4I+R1+cCzVMSiot5FMd7XOi7E2tv3Q/cSuoeHh8cewY5L6FMTJOEVV0QS7M6TZJfREtIUlYKKM8ExXxLp4o2z9Iv50adUatgmSytKIiiuctRhZhwAkI+Jo9WRQ/Tref8BcSGMxUly7CkIUfrmBZIECxmSUg8ckbHUvkYRep2WpNXscN6Tlor8XC3TuN4+R9LWq2ekpNvfPfQhAEAipcijiBPdPlR0cVGKFTTrJJFogbunm7SB1RXxmUumaKx1lmQ6bZFkCh2S7OaXxAUuzRGdPcq90RE+Dz14HwBgaUlyjKwwYRUEWm7gPCJNVbCCidp4JLXIOOPswpjSkXoslRU7KqKUpZ/RUdKODh6QIhKXOD3vmTdORW21xvZJSpMcxRdXpLzlcnOBIhfDAo2hl4tD7FeubYMpmvxMTrSZWpH2UWFoNGrL91F/wx6+fkmkz7U3aD+tqtS3DU5M02jLHNW4FN/Fy+cAAKVpkaT7+4jojqlScQHnQ9Ll7tLs6lpgsrVVEle/cp2er0uvi6vk4gJpO+999q9iO3zxc78Tff6Jn6X4RKPW0TC7qfPudDv30xRJ5rNV5Wa7ShpcRb0r8sPD3F9JpeyITxfl3FER5FEaXyUNv4ejxF9WFOwKu9rmaqR53vPwA9Gxw/cRWR2bmZDzmeAtqvw4IbugXr02y+PU7rh0LFDPUrCDuVw8PDw8PO4y+Be6h4eHxx7Bjptc5maJjGkptSiZItNJ0ohKWGsRARFyndG6qhX60ndIpR4fl8RJRw+SWmtbQnJ2WqRWnnyA1KfO7IXo2EgP+dMeHlMRiRVSy557WtR31KlP16a4inlGrh+2XRSkmDWaTDKZQFTY85yON0gwQQNJYjR5jUwWh4+J6afZctVetlfJtL/9GvtMdw+ICarVou+uLSv/2wSNJV/gbRCo7WDZX1cRjwl2to0p/+URrsJTyJH5amFe5sNljo0rtbLB/t+VsvLxZrLVqcPqdCTi9I9AmayssyUptdn5gOcLNJeBqnQzwDVN8088GrX1cuUm5XkcodBFZopGWcwycTarDI2ImW6EE56NchrV/X3iC51nc02gwkKXpyg+IX1ASPN8jvrmCLa1iphc2q9SJGqnIiY/O0B73DYkQjMdo3kYiLOPvzIzRrU5AzG55NgM1FB1bntT1I8DfUTSLqckmVwhpL1736jES7y1Qs/thXNvYztoUt7FUiTUFk4xAWo7smdcDdZam/pYK8t7IWCSMT/MkBPUAAAgAElEQVQk81wvk0mko1IdLy/TfDmTRzIta7YwQ2R4Xfnqnx2mOb2kIlBDNrvV2axnVZI6t/0HVHKzGJtQMj1iYnOxE8scsRrTtVOZdNXmSEfy385L2UvoHh4eHnsEOy6ht1zEXly6UmQpuNUWQi6f5WhC/reNy6/0mXP06/l//tq3oraf+BiRdA8fG4/aVstEqLZjTkqUqL92haSEZkVFELZJxOw0lPtVjX+J2W0rHhcJLM+pMOMlkW6CDv3St1Ui+7VOia9F/c7lRNp6+xxJZbkekdBTSZIYbWf73997jkuE4dwikTC5XrnG7BxJLa6QBwD09Q5z36gfKVXDNazTvUpVkZ6yLOlcvij5Wlx+nOIqzWVbkZ1xlpLbKjq1yZJOQ0UiOpFHBBjF5rLkY6ysQYvd3OqK2Mxz3c04S6lzcyIdptgdzYYilQ0MEPlXXtzsC5pht8Wckndi7L42rPLjHO8jiXWEK9/HVberTKZZ5TLXzaRYfVWk8NIs7RVTo7+2JJGXiQkaQz6QuSrkSCNydWABwLCEvi9BfZtUaWtnokmV5yvL/dVhmI4sD1tEwIayJXGMa8MeUtLkEvvQvvLfvh213TcutXQB4CM//jPR53SS0wQrrbtZIQ2h1VYELO8f00t7salq1JY491BbaY1N1lp1mucyk/EtltqvqvS5rq5wTGl3r3EBDxdBDgDVeXqGWjxm7VGY5HfVE0oaP8vrfXZSXGNjXJ/VuZUm1DsulaRnKZXQ+V3k2bxVeAndw8PDY4/gZgpcHADwuwBGQKLTZ621/9YY0wfgSwDGAUwA+FlrVb23m0Q2oF/klhJvEoYkKQuR9ob3k91xnoN8ViqqYESMJJI33hKXuXOXXwIAPHRUAnre/8BHAQCVNkmY1Zpco8FFLBp1kRMzXNx+rSSizDXOiXJlmpLi/w+PSdXz++8jsebin4iU1W6RNGSNSIdIscsSZ7trqcrfU5Nk10+lZWkevP89AIC4kqA3YnVV7ukKAdTKEvS0j+3IAznJ1zIwQBL66dPkHtdbEBvpQonmMp2T4K4S272XZsSNLpOhe+XZnpzPSR9dlru6Cq5ps0ZmdPGIwAUP8R5Yl5KP94KqdFBnDa6uCkoU8jSWAQ640cVAEnz9+VXZnquRpC2uhlF/WBzrVy52h8do/3Wr7HgZ1upK87THSnWxyLsslImiaDhxltYrSgMxnLPE7QGrNJFChvbTiLIBRzZoNUcd/g5TPmioLIDFNs1VWWlJLpFnqPKIrHAuF1eoJJcXEb2xRF9YnlXl1ZKc/bSmNZz1EvrQoGgzpUWaoxXVjyq717ZUm8tumOHyk1dm5Zl29u+OykiZLND+1FumzXla3P7XdupGhY4llAZSXCONqa60o1V2v61U2AVYZZVsM68zPT8n4+ulfRRk5bx9XAwln6VnolsFlHVHz4vMUYrt9q9+9c9xq7gZCb0N4B9Za08CeArA3zfG3Afg0wBetNYeB/Ai/9vDw8PDY4dwwxe6tXbGWvsafy4BeAvAGICPA/g8n/Z5AD/9o+qkh4eHh8eN8Y5IUWPMOIBHAXwPwLC1dgagl74xZug6X90WrTmKbkurqDWXwL6paLJmD5kKKkxKzi4IYRqy29PjD4pbWqlMJM+MJkQeJ9PMYtklvhfVjTUrtI2o1Ja1oQvLQi5eWWO1uUp/sxDXwMceItWqUpdItolrdK8rk6KOV2qkToZMUOYSKvKS/85MTURtgz1kTjiwX2oebkQnoVTZGt2zoMwl/UyyVlbFVW1xhgicDEfM1itiGlniIhYZpZsa7uf+e8alv2maw252A0ynxDzgCldULynymfX9QLk+pvlzi2tutpWLYoJtDBllgnKuj0llMhgaZHWWde9sWtYsctVUa7vGqWaR32xyyTDp2teQ+eht034qzl6L2hY5J8pKmUxLVxpCkJfZhHIw2xu17c/Q/tDkqUtj0uKautozNdVH5+dU1LCJM2mo0tC2OelsnXMZxVtiigpcRGlW9nq95QqEKDc9nvuQXVONKgwTlsjMuLQghF9rmNb7yHHJmbMR3/hTqR3f3UsEYlJFrDrSUkcNW17bex+lsU9dklTUy0x2WrU/0qtkGtHFNDopjo7l9c4oU0eTU+V2lFvrGrs5tpXJrM4OC871cmRUXJcPH6Pw8MbJ+6K20Rg9J8NqTp3pLsEmMasI0BrXhC2uidn3XSgpevOkqDEmD+APAPyStbZ4o/PV9543xrxijHmlWq3e+AseHh4eHreEm5LQjTEJ0Mv8C9baP+TmOWPMPpbO9wGY3+q71trPAvgsAIyOjm7yEQsqi3yezjpGn5tWuvfWGv1iLjHJtFoWaci2SVLqyogEluISarVQfnWXligAaa1Iv6YmJr/cJSbfmkZ+dBosPr32Q13lnq5XyNFv4Ze+IhLbyChJp8OjIvUdPE5E4/KyjG9igiSCC5fo17++Jvdc4Yx2raSIamlXEq2mapdtwGJNpKcUS7Xd/ZIzJMkSRN+oaBQXLpP0c/zEOADgzbck0Gpmlq53/ND+qO3ehygzYTwvfVvmLH1xzqVSU+5j6RHOdxMT967J8xM8FtEUclxysMN5W0xSJJksF6zIZsUVr3qB5rwrL1J4L7uQrbJmkY7J+Q2WSM9fluya1lAgz2M6aMz1h/NypFeEJJu+dJ7uPa+kVA7WOVUibe08ZEwNliLnktIWDtG+GFGesS0mLdtOPFMSeoeZ3ZZyL0xwPp+YyuzoMvZ12A0w1RYC3uVO6XSkzWlJ1khHOkzUJriMXkqR+HHO5VJviQtmx9LcZ7tknqHcQgHgS7/3QvR57BDtxX37RJHv7mIXQhVx0+G+Fwq0nn2KhF5gF9lApR1ttlhqD0XKb2XpXiEXqYAqaZiI0zMaV5kP11ZW+N6i5YY8z4vT9FqrqFwx2TTtz+VucTAo8P5s6bxFrCEE7B8aQrQey1qpLsqTeBd8Dm94CUPuCL8F4C1r7b9Wh14A8En+/EkAf7zxux4eHh4edw43I6G/H8AvAnjDGOOql/5vAP4FgC8bYz4F4CqAv/mj6aKHh4eHx83ghi90a+1/BbbN6/iR2+3AGKe/DBUjUG9w1KZSHastrvnZQ6aDisrdUC6zr21RTCO1IqlgY/skv0uSzSRzHIFXbAuBN1Nk/+m0qn/Juu6SrgtZml93/+MxKZDwze+SWq4jXE+cIDX76FExdbznMerT+54g3+aZCYkcnLhE15+ZlvEd20dmmytvSYrcjWgpdTjkuaqWRIVcW6YxHzlwKGorc96QBkeKDgyLaeR9Tz8JABhUEavOlHRJ+aFX2Cfc8N9KRVTTQp5U0pFDYtpKcS3FZFy2VImJrXab+j06dliNi66ra4r+4DT5NMeUKh1nM42rR9pSRROW2SRy6YpEj8Y5mvIxbEaHCaups2ek31N0z7wiKCeYRLtsaB8NPPaE9JujGS+ekuIUyUky+aUCIZotz72roBCoFMkdR/4pE4Nb0UAp12xRQoOfl5ZK8dth00JL5S4Bz6XOd+P8/F0d0ISKRAWb0WrqeXQR3rpWqcoOS+fX5BoLs7R351XBFMPrl9O+2EyqT80QkT44IM+vs2aEKj4lZmjdY2pOU2yyTbs8SooMR5PWNq7Or/MQ6mosbnvu52fvzKuSevnKKSpi8Y28mJQKnNY7X5D3RzcX/xjZT8/cocPy7D31BJHJo/vEd98VI/lP56/gVuEjRT08PDz2CHY8l4sjszpKQs9k6FcrUBKBI34WOCLyvv3yazfLZbmM+tWtcmL8fJdI4UFUNZwktqtT4tLYzWXbkPxA1DazSL/m0/PnozYbklQbZ8L2w8+8Nzr2VY6u/Pa3JdfJNXZXfO1VKVd1gIto3HsvkTfHjohE2tVLRNvKi29EbXWWak4ekajUhQ0eQzFdgZyzN9YrigjjzHBrRflem6WxNBOwsbQq18cEkQ1EGpqYpHloqnwchTytX8DiWTYjRFGGq5yXV4SYDrJ0Xq/KBBljkrWyRn1cWhUnqnyB5ipUoYCunFlMu4GxVlfmzHZFVQDlCkuHgXKZq9Tl+EY4t8niqkQp9jWoTyklTU40OYrw5OMAgEOPvic61ihxFGlN1mD+Fcp7EqqoQzBRVmYCNK7cEV11+7gau+tbR0nyIUugFT5W1Xlv3F9FWDp3ulhHu9hF4i/9UeS2ZRe+lnKrK7t9pPLSJPpFOgUAxTviqUeJfC7kpHFqitZqpSR70ml4cxy5Oj2lyig6l9dAk5xMLsZkLyQ5MjPDkbZJ5UqbZUm6oEj2NHc0o/ZTNkfHh++h98z97DgAAG+8TlrXW2fORm2LS7SO84o0x1nKRGm4v0alEf1KNz03Y6oQy/4D5IDwwMntXUFvBC+he3h4eOwR+Be6h4eHxx7BjptcSqXNMUqWzStJRdpwPhwEXLAio9SuQU47WVIpWQ0nuinkJVJvcYVMM87HNJ0RtWutTG1f+NKrUdvsLKnUykqBrjwn1OKEU7/9m78RHVstkZ94XJF1lquWN9rSj8tXaFxXJoloe31AVPsDY+MAgFpdVN4UO6gmw+3rYCZUAYOeLJGXvQUxa5Q7pMoqjR5DTNqMHSJy9tKkkDEJvly+R9TVNldbzybFZBC6Qhh1lwpYTBKuNmcgvCqafJ5Onts9RMRTtU6mkVCZAtYqNKdzSpVtcxLlfF7I1uU18iWuFqmPhezm9Lz9vTIf1aZyBt8Aw6aiVlYSjVUWmGxVm2GVzSSpfprvelPMGv3D5G/dPSgxCdM892sqJWw3k4u1yAlZFafgv5prdHu3o4hSd9c6f6OuXRicaUYlgHN8cS6TVm303SYnSKsqErrKyd5aqiexfhpfTvliW+VTDQDD+yXZWzLD+0hVuDh8jObmeEyIwXrNkZY838p/vsLFLq7NCPGe5mRYOsFXg8nYFU5NHFPvkSpfv6VCco+dOE7978h5vX1kEsnz+Ap5eVccOESmz6VFlcyLIz6rZTFB1XkOXfRroOrATl2b47FIgq+XX34FAPDAr3iTi4eHh8dfeuy4hO6iy+Iq+XvooufSKpcLn9fh09qqpP1akX4dS1WRak2KyI+kcr8qRSQhiZ8ffObp6Nj3X/6vAICv/JG4mWWzJAHGM+LOl06RhJ5kyXR2RvKUNEOOBFNVzG1Av87rcnSwZORcz2bnxB3x2jVyebxHFeZocNm9jlkfiadxcFSI1RSLxK4sHAB0uBhEK5RotSLntdgXEjFz9Ng90bFLV4jwqTZE4siwdjK9LFJFg9OnujJyubyQ1TUm08b6hcxNJrhIxpSkNa4yudhiyfXx94n737e/TURiRRUfSGapHwUlHfaze9tQP0mF4weOyzjXaJxltT8uXRVCfCMaLKnVcqJatCyvuyp2UmdXyRP7qEDCQK9oYUXOF1SrqchjLvIwq6TlBEd8lng7r9MbWGBUqUtgnISuTmuyVN9gsrOqS4U4Sb4jz0GHSdS40rRi7D7ppMqKEvXmmczrG5bSeYUTx+gavfJsNErrI5mfeFyRxLz/UikhHtus4dRV6bdYmta0HdC8GTXSPi560TUk69jPaX6tdnvmHE2tWXJE6Nkn/e4dO0HfGxd348efJkeImIpQLnHBjwa7sPaPiPukc+1sKC16laOK5+fk2Viap3fDwjRpl5OXJ6JjU9y3xSXRzivFrQoivjN4Cd3Dw8Njj8C/0D08PDz2CHbc5FJ3dQUVCdPlqpAEm1O3tljdqVVEPSmxul9X0YGdkElARbqWuIJJo0X3WlMVSty96laIs0aZo+Yacp6rTB/yvULl/w1O4WlMU51PKltbMVUNJv1yGSIQbUPIphwTtf090o8EJ1hKpXXFIk36AQFkroqs/pXWhERNOqdgZfpxVdTn5snMM6iSiq1y/cbBXvEttpH6K2u1xtG5GU6iFcRUSuIyJ3WKSz+anBBqdlpU0yRXOUozCZntkrH0DpAK3tV7MmqrN94CAOSUL/HgIKnE/RzXEDdibjJMsvf3yZxeviyxAhuxxmaSuk4IlqTxpdqytnVOeLbI8x2kJQoyxyRaT5eYJC7w9pxXCZzivCALkclFl4Zf94f/4fzQlV8+t7kdUVMmvyYveyeUdTFcHzWWlMffsunMcOxHWpmzskxaF8bEdAYmexvKTLERmbjs0QSbsXpUSmcXe9JOiimxzjVs2y2a77ZKmJVM0ndNXBwAli2tc6Mue6zdoPV75r0PAQAefO4T0bF7TpA5L6580+ucvrejzLNZfubibCLSKXtdBaTAiIntwH4yuwUPPRC1WV6HkCssdVRcTYPvubgoJtviAj1zP/z+m7hVeAndw8PDY49gxyX0Mrv5KIEDLptmUBfJ1ckXTXY3a7ZE4nAeaG0TV230a5tXPl8lvlebI+l+cOo70bFam4srKOnGOFcoTdoYkg4sd7JlRWILWQpxqWoBIAYmQHUZ9QZHaOa4eIMKsBvop/PrS0KU7j9CbkyFpLgEAiVoVJTG0uYoxWJHtJN4msaQUhF1GY7qTGeoP5OXzkXH9rFbmksVCgCNJpOoPap2JNfEdAUG4kZF7HGq0lJdNBxXX7RXSf4PPPggAODCJZK8z50RYrqH+xhTGsiBXiKserpEQhrhIgaVGs3Lak3mr1J2hJya6O2yEwGotqiPpYRIcUscYdhZEolqniW6oxkiT5uqdmqdK6YorzuUeY9NKTe6ImumxZCjmJXrX8Ck4fquhnye7Mk2f8dFlrZ0iXpua6poU+deWFOFJRypHXeSvJKCS1maB11QxHQ2R5turHjbUcdiXOW+o1Pshk7bVTVCmYCN0soqLSLGEn1cRVwa7rdpCNl/4qGnAAA/9vO/CAAo9Iq2EXbc+0PG7rTXtipo4qJNU9207lYtpHOr1U6alseia9k67SLk901b5dOx/H5KJ1Wt3NSGZDi3AC+he3h4eOwR7LiEnuFffV0Oq8nFD7TEXWYXoTiXRIvFlY2UfytzGWlzCdlmVabEEgcV1Fgd0K6Spu3cy+RX1AUkmHUV1p2djY8pe3LA4lAiIbJKyJJXXOWJyOfceTTOkW6Rhj7w8L0AgHpR7Hl9rpK4VmM2YHFeJMexPgoYShfkuhcvkxvicK8Ee+QK1M9i2X1Xrt/PUrjRPnPsMtcoi8R95MBR/ibJBmFHucyx5JPLSD8SLIUk4iI1Ocm/zRkslcCGqgvYKIr01MUS8dFD4iLZx5zD3ALl/qioAhoN1rCmlItpWWWF3IiQxSyr3RbZLbJUE1ttie2201ym7CPPSPLReovG8v+d+oG08VTOq+iuVd4zDVdCTYl4JpLQtYzObrDqeXFSvXONDZU06fQatYURsi23f1Bc8VzgzPRlyqTZVHttjp8bqJwrBY48y6x3tFyHARVQ5qimVlvmvdbYLOlG0nfM+WxqGz3b/jXNwNzaoTHJifIL/+On6LoBZ59U9nWXeFEXyQi4FOS6Yh089+5esZTSujmoUeefctliA8WPROvn7qXuucZukSlV1aKfNc6Xv/VD3Cq8hO7h4eGxR+Bf6B4eHh57BDc0uRhj0gBeApDi83/fWvtPjTGHAXwRQB+A1wD8orV2+5yk216f1JCMilpzVcAbKj9Dh4mFDJMkeaUOBy5STqX8DNm9rFwWdavFunyDzQK1hkovy8xqbAu2bL1SyX1y2pZSu1zBhbgij5wFIhYXdWuUc6jkUnTleEkIzlHOT1K2K1FbrUKfE0lxu9uIUHcyTb/T82Wpdxqwe5k2VfUO0r0WFilqLZkUU9HiEplVNIlVZlNLKqW2TZNJ0YCuH4QiIwyxGSSekjlaWWWzSlvWamWFTCGORAtbMleZLKnDrboiAd26KQK7zutdZfU6pVwOr81N8r0VcXydEutpdg/tUpGfzj01o8xHKJLafPUKmSn+7E/VXuP9OjsnaZP7h8mMlVZ1MuOsvhfibFowm1V2bZRw9XZ1wY82f27zJmgrgtCZX0yoHAz4cqsrEtnZqLtcP2xqUG59iQTNZUy9LoKmIyi3N7mcPifRuG7MWZWiOcFphI2ujMHPkCsYEQYqArRJ6xdXhG2Txzp+VCKli1Ua1xqvT1q9WwI2feqnPMH7vlqW/bGyTO64B9msF1PmWWeqNdo8xvOm8zg500ySz9fH2vxW6VLuycX5LcsyvyPcjITeAPCstfZhAI8AeM4Y8xSAfwng31hrjwNYAfCp2+6Nh4eHh8ct42ZK0FkA7qcrwf9ZAM8C+Hlu/zyAfwbgNzZ+/0Zw1dzrSrpO8K9of1Z+iXv5F7DFknRcVflOsLgcKonNka09KmOekwRaaxy8oNyfwIEjgfolbjVdkIPK4uiIp3AzQekIEe2GFWcypV+VphplyW+kl6Sg+oL8MtfLRAKGumQY32p5WfI+ALpIAnDsHsldcmWagmYuL12I2hIcgNGTFjJyZYVc+4olkkbSSgNoVmjMrZbK/cK5Ou47eSJqm+MCBAFvpXtPSI6MoRGSSM9dlmIdzr2xoVzEnLSZZbKz1FTZ9DjvTWpQJMblBdIUFpbFNTHPOWoSLFmWKkLgTXJmu4RyQ4yl18+fRp6DuzJKSs3zPm0qiWqUpd8256xpqwyONZak9w2LlJ9kidTNASBkZci5iTTvbZn4XLfXeAPqIJX2hsCVsCXPkqtoP78oZRFdxfu6IngrnA/JZUMcGByKjnUVKIgoqwLb0k7qNNsT9dPLOjsoS/6K0Uy7LKLqmcuwG1+BSeuYcowwLK3bgmhf+RT1afKcuLoWuknzdFppXAU/uXcLFHnpAvamJ6eitkvn6dlx7wOj3i3uOV9XApHP06So++wkee2EMXeNNLeWClQrr27OPPtOcVM2dGNMjAtEzwP4OoCLAFatjd46UwDGtvnu88aYV4wxr1Q3VNnx8PDw8Hj3cFMvdGttx1r7CID9AJ4EcHKr07b57mettU9Ya5/IKrumh4eHh8e7i3fkh26tXTXGfBPAUwB6jDFxltL3A5i+7pe3v6b7ELWlmXhKKvKtweSHi1ZbR2pw/b91eTDWnPqi/HVZJe3hWoNx9XPWZJKs0Vb3ZPWwHVcFK9z5TNwGSu1y/sJtlRPCtVllYkiwiaGH1cXs2Kg6n5Piq347zaaq/GmRlJSgAPDGOTFrTM+R6pjvF7NCd5bMGf1DYjJYK5MJxUXdNlWq3J4uun5W5URJcj3QiamJqM1yOtmBASJ6m8pU5PjRw4fFHFSu0LqcP6/rtLIvNpvdMmkxtaX5cyKj1NsY9enCpatRW2OK9ke1RXOV0KQef66q9Ln5/PZbP+XUbPV4uHqTLVWXtMVxCklmGbVppMFl440yjbjsx/q6Lpgy5Pw/4Ra+zTri0vn7Wx0b4UwtLoJR5T/pzZGJTQdo1thUVC2LxuyenFiOxpcviGmuN0v5aGIZlYOGzZWBMltqKyEAPPXovZuub7eIpbBb5K9xrghG1U4NjKsfqsjIJLWtrYq55LvfehEAMDhGMRLdPZJPx+XC0fVoncklpuTb0VEyOMzPznK3NjtLJJQpR78Hols5P3Q215SL8nx97Y/+gO8tQu4//+e/CgD4zbc/t+laN4sbSujGmEFjTA9/zgD4KIC3AHwDwN/g0z4J4I9vuRceHh4eHreNm5HQ9wH4vDEmBvoB+LK19qvGmDMAvmiM+T8AvA7gt26lA4nEZjLB/bI1akLItTn3QpyJpURcfh2dVJsMtMsckZBdLSXtJZjbNXTdnHIfq3L2xIqSrqtcMqqt3J7aLA0l+Bc5UG5mTkLqKBI1FXPRbXJejbMQNqv069yVV5GlkXuc5G0J3NhVPyBTAwBYrgtB2N1PRFy3KnBR4OIUQVJpLOwKWOWSXUuLQrqurZL0Nj4uEXiZHF1jrSQ3jwinJM1VRUUCXp0lt7VEoLZZ6IoDSD+OHDnK9yQCT8+f+2ZPl5DKg0yQXrkmZenW2FUtzUR6TmU5nF8kzSamSMtWZ322So0Y7y2jowmjY3JegvW1hNksFyWdLqfdI1kTCVWpMydoWyTchwgu54qW2kWhVRI6a3zuvFBpBdk87aN8n5CzLrNpRbnpuc/5PprnfhVRbNj3NlRFZZxzgH5uNzowapk28vDTyU54CIEiVt3RIHCugeq9EBGT8tw2Wkw4KvfGJmvnV6coV1NGaxtchlC7nyY4z4x2Q0yyM4Od2zyWBD+HSfU82ihyfHOkb4znaE6Vm3vx698DABw/Lu6WL/3FV3G7uBkvl1MAHt2i/RLInu7h4eHhcRfAR4p6eHh47BHseHKu5WVSs3tVVF7gkt+oFJdOBeuwaaStfG2jNKMxUbu6WPVOxbXfK6leGSYjKy05f5kJs1VFnJVa7Ceu0ow6MhQcXRbqKD7+q5N5JVjdymaE/HC1T2dmKUKymtvsEz3cJyqvq0HaqG8fiBtLqnu6CLy2Mks1qa1SkfE1KzQ3yzNlHpKosgN9RKLGU/Kbv8Y+8mnlrZRignKtTH7Oxbr40mayNN9xK/3I8rwNKj/nKid9CjouGZUiptlX2uZ1IRH6fPiImIOS8/SdpRKZfGZXRb2tx+gamX4ZXyqx/dZ3UYHtdnvTMa2Wuys4f2RN9yV4D6yjAHkINqb2DJtOrN1cKGIrP/QQ4bpjABB0OP7BmVwUie9SzibSqr4sm1zyeZWO2Q7zALcYZ3LzXLXYLLYVyRndW/ldu/PWkYuuTfuy8/Nit7BYRY3rCsgQ2h0doUmtea49e2VCipmceYPMhZmcjL2LzXmBsqcZXlPnt659zlNsagl0qm3+vM5E5LrrYgdUdO+BI+M0PlXE59d/8/MAgA8/8xO4VXgJ3cPDw2OPwFzvF/bdxujoqH3++efv2P08PDw89gI+85nPvGqtfeJG53kJ3cPDw2OPwL/QPTw8PPYI/Avdw8PDY4/Av9A9PDw89gjuKClqjFkAUAGweKNz73IMYHePYbf3H9j9Y9jt/Qd2/xh2U/8PWWsHb3TSHX2hA4Ax5pWbYWvvZuz2Mez2/pww4ngAAATmSURBVAO7fwy7vf/A7h/Dbu//VvAmFw8PD489Av9C9/Dw8Ngj2IkX+md34J7vNnb7GHZ7/4HdP4bd3n9g949ht/d/E+64Dd3Dw8PD40cDb3Lx8PDw2CO4oy90Y8xzxpizxpgLxphP38l73wqMMQeMMd8wxrxljDltjPmH3N5njPm6MeY8/+290bV2Elzk+3VjzFf534eNMd/j/n/JGLM53eNdBGNMjzHm940xb/NavG8XrsH/ynvoTWPM7xlj0nfzOhhjftsYM2+MeVO1bTnnhvDv+Lk+ZYx5bOd6LthmDP8X76NTxpj/6Kqx8bFf5jGcNcb8tZ3p9e3hjr3QueLRrwH4GID7AHzCGHPfnbr/LaIN4B9Za0+C6qj+fe7zpwG8aK09DuBF/vfdjH8IKhvo8C8B/Bvu/wqAT+1Ir24e/xbAn1lr7wXwMGgsu2YNjDFjAP4XAE9Yax8Aldz5Odzd6/A5AM9taNtuzj8G4Dj/9zyA37hDfbwRPofNY/g6gAestQ8BOAfglwGAn+ufA3A/f+fX+Z21q3AnJfQnAVyw1l6y1jYBfBHAx+/g/d8xrLUz1trX+HMJ9CIZA/X783za5wH89M708MYwxuwH8BMA/gP/2wB4FsDv8yl3e/+7ADwDLnForW1aa1exi9aAEQeQMcbEAWQBzOAuXgdr7UsAljc0bzfnHwfwu5bwXVAB+X3YYWw1Bmvt17iwPQB8F1TgHqAxfNFa27DWXgZwAbuwItudfKGPAZhU/57itl0BY8w4qBTf9wAMW2tnAHrpAxja/ps7jl8F8I8BuFIB/QBW1aa+29fhCIAFAL/DZqP/YIzJYRetgbX2GoB/BeAq6EW+BuBV7K51ALaf8936bP8dAH/Kn3frGNbhTr7QzRZtu8LFxhiTB/AHAH7JWlu80fl3C4wxPwlg3lr7qm7e4tS7eR3iAB4D8BvW2kdBqSPuWvPKVmBb88cBHAYwCiAHMlNsxN28DtfDbttTMMb8Csik+gXXtMVpd/UYtsKdfKFPATig/r0fwPQdvP8twRiTAL3Mv2Ct/UNunnMqJf+d36n+3QDvB/BTxpgJkInrWZDE3sOqP3D3r8MUgClr7ff4378PesHvljUAgI8CuGytXbDWtgD8IYCnsbvWAdh+znfVs22M+SSAnwTwC1b8tnfVGLbDnXyhvwzgODP7SRAB8cIdvP87BtubfwvAW9baf60OvQDgk/z5kwD++E737WZgrf1la+1+a+04aL7/i7X2FwB8A8Df4NPu2v4DgLV2FsCkMeYEN30EwBnskjVgXAXwlDEmy3vKjWHXrANjuzl/AcDfYm+XpwCsOdPM3QZjzHMA/gmAn7LWVtWhFwD8nDEmZYw5DCJ4v78TfbwtWGvv2H8AfhzELF8E8Ct38t632N8PgNSuUwB+wP/9OMgO/SKA8/y3b6f7ehNj+RCAr/LnI6DNegHAVwCkdrp/N+j7IwBe4XX4IwC9u20NAHwGwNsA3gTwfwNI3c3rAOD3QPb+Fkh6/dR2cw4yV/waP9dvgLx57tYxXADZyt3z/O/V+b/CYzgL4GM73f9b+c9Hinp4eHjsEfhIUQ8PD489Av9C9/Dw8Ngj8C90Dw8Pjz0C/0L38PDw2CPwL3QPDw+PPQL/Qvfw8PDYI/AvdA8PD489Av9C9/Dw8Ngj+P8B4YzKGaRtpggAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['truck', 'deer', 'car', 'plane']\n"
     ]
    }
   ],
   "source": [
    "train_iter = iter(trainloader)     # 生成针对trainloader的迭代器！\n",
    "images, labels = train_iter.next()\n",
    "\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print([classes[labels[j]] for j in range(4)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Define a Convolutional Neural Network\n",
    "\n",
    "Almost same with 3.1, except that now we take 3-channel images instead of 1-channel images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):  # nn.Module: Base class for all neural network modules. Your models should also subclass this class.\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=5, stride=1, padding=0)\n",
    "        self.pool = nn.MaxPool2d(2, 2)      # 与3.1不同，maxpooling此时也可定义在__init__里，forward里只使用这一个shared的Layer!\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(in_features=16 * 5 * 5, out_features=120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "    \n",
    "    # Structure: input-->conv1-->relu-->pool-->conv2-->relu-->pool-->view-->fc1-->relu-->fc2-->relu-->fc3\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)                      # 疑惑：Softmax或Sigmoid函数在哪里？？？\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Net()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.3 Define a Loss Function and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "# 在训练Loop外面事先定义后Loss和Optimizer，在训练Loop中可以多次使用\n",
    "criterion = nn.CrossEntropyLoss()                                # Classification Cross-Entropy Loss\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)  # SGD with momentum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.4 Train the Network\n",
    "\n",
    "Things start to get interesting. We simply loop over our data iterator, and feed the inputs to the network and optimize."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1  i:  2000  avg_loss: 2.178  current_loss: 1.712\n",
      "Epoch: 1  i:  4000  avg_loss: 3.980  current_loss: 0.763\n",
      "Epoch: 1  i:  6000  avg_loss: 5.624  current_loss: 2.094\n",
      "Epoch: 1  i:  8000  avg_loss: 7.187  current_loss: 1.520\n",
      "Epoch: 1  i: 10000  avg_loss: 8.702  current_loss: 2.115\n",
      "Epoch: 1  i: 12000  avg_loss: 10.141  current_loss: 1.325\n",
      "Epoch: 2  i:  2000  avg_loss: 1.379  current_loss: 1.600\n",
      "Epoch: 2  i:  4000  avg_loss: 2.718  current_loss: 1.659\n",
      "Epoch: 2  i:  6000  avg_loss: 4.073  current_loss: 1.359\n",
      "Epoch: 2  i:  8000  avg_loss: 5.372  current_loss: 1.438\n",
      "Epoch: 2  i: 10000  avg_loss: 6.658  current_loss: 1.734\n",
      "Epoch: 2  i: 12000  avg_loss: 7.932  current_loss: 1.163\n"
     ]
    }
   ],
   "source": [
    "# 手动编写训练过程，在Keras里只有一行：model.fit(x, y, epochs, optimizer, )\n",
    "epochs = 2\n",
    "for epoch in range(epochs):   # 手动遍历epochs次整个训练集\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader):  # 对于每次遍历，手动指定每次的mini-batch的data，及后续如何使用data来计算loss，让loss进行backward，随后optimize\n",
    "        inputs, labels = data\n",
    "        optimizer.zero_grad()               # zero the parameter gradients\n",
    "        \n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()  # 当前2000个mini-batches的loss之和\n",
    "        if i % 2000 == 1999:         # print every 2000 mini-batches\n",
    "            print('Epoch: %d  i: %5d  avg_loss: %.3f  current_loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000, loss.item()))\n",
    "            runnning_loss = 0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.5 Test the Network on the Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJztfWuQXMd13tf3zmtnZp9YLIDFg3gQfEICKVEUKVmKTEllyXGZ/mG55CgOq6wKXRW5bKdcZctxUo5S+WFXUnaciqOEkRXRiUoSLSuR4liyLEoyZceiCFIUCIHEk3gD+37Pe6bz45zuc2bn7uwCoLDYdX9VqB30vdO3b3ffO+ec7zyMtRYBAQEBARsf0XoPICAgICDgjUF4oQcEBARsEoQXekBAQMAmQXihBwQEBGwShBd6QEBAwCZBeKEHBAQEbBKEF3pAQEDAJsFNvdCNMR8wxpwwxpw2xnz8jRpUQEBAQMD1w9xoYJExJgZwEsD7AVwC8AKAn7fWHn/jhhcQEBAQsFakbuK7DwM4ba09CwDGmM8DeBzAii/0fD5vBwYGbuKSAQEBAX//cPXq1Ulr7dbVzruZF/pOABfV/y8BeHu3LwwMDODJJ5+8iUsGBAQE/P3DJz7xifNrOe9mbOgmoa3DfmOMedIYc8QYc6RUKt3E5QICAgICuuFmXuiXAOxW/98F4Mryk6y1T1lrH7LWPpTP52/icgEBAQEB3XAzL/QXABw0xuwzxmQAfBjAV96YYQUEBAQEXC9u2IZurW0YY34ZwF8CiAF82lr7w+vt539++tMAgHq97tucaaZWq/m2OOLfnhZZdaJIfovc58hIWxMtOj1u+bZMmqxEcUx/myb2x0xPHwAglSn4tkK2h/5msr7twtkzAABbp7Fl0zKFNTTpWMJ9am8iN950KkPX5L90Ho23VpP5aDabPEjp72O//LG2/n/6XQ/IvXD/mYz0m45onJGRTuI4bhtPFMuxKKLPRp3v7iGKZN4sH7f+e3pd+DwrbYbXSJ+3fBz6mvJZj83yNZu+rdVqtf3V+8ntIz2nru1zX/0OluM7r+S5f7Vmfm8lrS7PQZvHmG07RsdjLIe7P+PPkz5Mi8Zrooa0RXR/tiXzZ1u0ttbNqdHj6OwXltuaakN1c3bj/iLdr+m8v3e8ud2k+hu/8a/8Z95OybOnntvLl4mWWyotAQAOHbrXH+NHH/WKvBfq9SoAoFSWa89OTwEAFhfm6P+z8/7YwXuov23bd/q2ZlPeESuh1eqcP73cvO3a9kCjSZ+by/YmXbPJ5zTU+XT8s0//p1XHsxJuhhSFtfYvAPzFzfQREBAQEPDG4KZe6G8EFhYWACz7ZWs0Otvq1BZHnVYiJ+VEsRxrNejXLg2Rirb2k8tko0mSz+xS2R9LG5qKrcPiGdRX7AUATFy9Jv3yr2zsJEclTUZO4teCD9+DvhcnkfpxR/KFVstJxtJHmrWAriEDehxRpxTsJUEtcXeMsVMab78E9xHpPvgekvq3Tpq0qs1JJElSuBtH19vzn+NUp+TvjmktIpMmrSuORKKPI5HyOq6VcpqFktwSNAUHGa8eZOfZxmsqCX34c6QtNjTeWmnKt9VrJLlme/p9Wyo3BABoOClYSbzSsZpv65oSzksES5q20XHEdN6KgpZ8WTtWe6wVuedWPzD0nSw/I89981v+0OT0JACgWq34tlKZJPR0Ou3b0mnWfFlDrVZFM7s8RhrZ448/7tsyrIHrfbd8/ycpX+5Z1edrSX651ug17YRj9FmO3yhC6H9AQEDAJkF4oQcEBARsEqy7ycWRU0mEQZvJYJmpJUl90fAEniJ+shGpYE4Zj1H1x/I5IsIKhaJvS6VIjauUxTTjVTHWNZ15CAAyhR4ea1c91JsYnCmiZeVenPrZkxciNutVwpX7dWYcoH3e/LgTSDpvRPDEXHckmW1ip7Ynkpd8n9HKRCLQqd7q9fTmoLbj9L9G1XacJ99V/bdiPkertysTYcaZxNR8mwTbgr96Ql9JZK7svISZ5vEbZaZYmp8FAFw984pvqy2R+aV/6x7ftvPOB3ncbu+uspLW/VmrPOfGlupo62ariZRJrMrPSVU9LzX+brUiJpQrlswjxydeBwC88I1v+GN3j5CX9L79B3zbyA5yZigW5bnN5HI0Xve8GzHHXL5E8TkvvfR93/b2hx/mO+liTrOdbXrfJplQlptbk863CSaam0GQ0AMCAgI2CdZdQne/WlrC1NK3g5f2ElyM9HcdWk1HXsqxIrshNiP6u1hXbmkpmoqG6j82JE3UlAuckF2dUm3KjUNJ6M59LmmMjp3K5cS90BFV2uWwUHABWSv//ur+vXTYRiR2StdJUmc3JEm/YI3C8DqmUsql0YuCsp5Oe9HSMtQ32v8CjlhrF+I7CWkv4XKbdkVrtWgdW2pfaZfY5fBj1POdMFeOXPSCfIK3YJvU14V47NSfgFp1EQBQmh/zbVmQe97irBD18wskyReHSVrVc9Wpj2ko18eVh4bu0vjKe+jb3/mm/zxlSRueUOs+PUfjnjl9xrct8PEfjl0AAKRjraUT+ZtNK3K7SNJ4Qe07v0/Z7bNh5fl10vvswoxvO3+BtIHde0Ty98P00rWaKy+hd0rj2ruxtUxrbCdAmSjVEn2r+yqsBUFCDwgICNgkCC/0gICAgE2CdTe5JBF4Dt18obWJwfmgtpMKrCopVbnGvul1jjBsqWPFQVLnir1CrpTYR15HHcbs3xxx/ymlErpxNJUq5saZFCnqUgk3GkolZFNLkX3gASDF5FKttrKfairBpKMjZ52feLtvevvftcIqE4rTMZsREbfzi3IvU5NE4JXZdxoA7ti5AwDQ19PT2Z9JUu15vbUpx5HJXSL8ojZrietPzncRl0kwHFVrrD4/KVK0xed1k4uU33Uz6uximWmmqWSsQnEYALBlaJtvsxVKl1SqLfq2palLAIC+LTvcN1X/HNMBZTbkiFU9BxL4mUSQtzsC6IF3q6fw5D/7Jf+5ViHHgqWyOCIsLZH5yEVdAwDYDFocpHsfvXOXP3Ry/zQA4PXjR31bcTfd8/ZROe8XH/wJAECanRq0+c3FsxQLfb7t7Dmav6FBiUEpFMjPv+G3TicBb9tMKMtJ+U5TS/IxeZaagRQNCAgICHBYdwndQRN0Topsc8VzEoHp/LXzboBKWnC/dlklCRZY+h6fJckxZoIEAFosAVol2tVr9GvebHS6SDriLJGQTRibbnPEoRtvjxrj0NBQR7/uvGZT3LuWI04guHSbz82ifsKjZe6V7dGYCUyfy8NitLZBc3N5niSwoyck4WaDI/RMSrmq1ScAAHfv3uHb8nnahq2mkyKVVMv3XtF5ffhvWkUYLpea9J054apdAFpZLYk4aliHbSbPR6OzrQOalnQSuu6X19b1pEj8Qv8IjWd0v2+7cOoqndcUV9rKPEmY9RrlKclkh/yxliOVjdqT/NhHbffixtP+Vx9MItG7udpdPPm6/5zie49T4o5byNLzl+2VojepHtJQBwZIK6mXRKJ/9W9eBAC8/X3v8m13Z0YBAN858oJvm9z/EABg+xDtMU1e+qhUFeWZy5M0fuLkSd92+PCD/F0XNZzgItvW1upoW5uEHtwWAwICAgISEF7oAQEBAZsE625yWasvtFdHWKXRyXiceaKuyBWnFuVyYs7gbJaosXqUyQsB2j84SP2mpN+5uZm2awJCiqYzqba/gPhd60jRCC41bWdaWTfuvj4hQLNZUUk7sbLJRZOiTu1LIkC1eUASWnWugftuG+nlI1yVHzCnJK5XyQQwNzvnjw0NEMlUgJhcJqbpeMmKKv2mvXcAAHrZH//qmPhYn2Yf5aEtYkbYz+e3+Qb7yLtOtbVlO1XebpB77zSdtTv3u7Vfm/9wKnZEvbS5PROxCaW8tCDdc1RlOqP2TorMFC2VCnipRATp7Bwlr9q+Q+rOWB6jicS8CEvzHGnToCN4feSvGriPLO28zy4+Ddi5U0xFqRwRtT0Ftde54E2uKM9oPk+fpyfoXhampcrl0gzvLcXJp3gPlKamfVvTmzLpWVpYEALZDVgn+Wvyfq6qfX3hwjkAwJ7d+9v6BHSU5xtBinZGyN8MgoQeEBAQsEmwqoRujPk0gJ8CMG6tPcRtQwC+AGAvgHMAfs5aO7NSH10HwNKqjuJLuWjJtERLNth10OVO0RJ6KuVINR3lSX3oX8C5CrnPVVmYHewTCd37JynpulGn7/ZApBtHzuayLKH3yjhc2tJ6Q6cZ7YyMdDliYpbuM4qczXE+mDYi1hW9UO6NyxFpctYTg5qQ65Ta/biXRVlqJBJh+jNLJll2gcsYGXeeSa9dSks6u0Bzc25GtoutUFs/ayevnjzuj/VylOy9IyO+LcX3oCWabnkz1ipBO/i57JI7BxBptnvmWX2U1q+l14VJutkJIjtfPy7kXor30Z6d4pK3dcdeAMDF86ekV3YLLc+TZpMaFcK0lSBdG5fbRKUTlm3vomQVjFVH1o5Dbz7kP//QEaQN0aJbDX5Ga7J3azE9E5Uy3VOkdpsjcaOy9FGrk9Zqs9LHJ5/5bwCAd73tnQCA9z/wmAyK1+PMZZm/XdspL05RFbeZGCOJf6CPCNt8QdIVN7sQmk317DdY4nfvJe0+WWdX3YbSCuq3SEL/DIAPLGv7OIBnrbUHATzL/w8ICAgIWEesKqFba58zxuxd1vw4gPfw56cBfBvAb97IAJZn/NOf28IYutjanS1L25F7B8hWt2e3ZKWbb9CvfqtBAQ2DA4P+WB9nWZyemvBtQ/3UNtAvAQcXzpwGAPSwrS/OyxQ6ZUBrD+4m9K+5G2eSvdcFAzVUMQEnbTYaK/+CJ5Xkay9mwblt4pV/w9fKZ+jiEc6gaWs0p0a5Vo6NUw6SWHEVDVcSsCF9nDhNbncl5iz6h0RSevTwYQDt2fSazr2xS1BLN14A6G5P9+etKti7fbryvBndCdt7W8o10TK/kGYROV2XcmmNCtnTxydk3G86TJkBq4ovOn+WMge2Fmkem/OX/bHZEvVvlGvs8FYOmlHbyS7T3LoFDGl02zPaNm75Yo2snN9kt2AY4VNynPXU8V31ugyyyPZ3neF0cnac+leZHY+cosAjF1T42OEfl/H6AC7Rdo+epPk7dPDNvm2gSJL56+cop8zBg3f7Y07D0nvIPZtJNnH3N6mYhb6/Wpfne624URv6NmvtVQDgvyOrnB8QEBAQ8CPGj5wUNcY8aYw5Yow54oo/BwQEBAS88bhRt8UxY8wOa+1VY8wOAOMrnWitfQrAUwAwOjraocclRnkuU1WWH1/pmFEml342p2QLQjju7ifXt2iWyLdmTcwDrQapfT1puc4j73orAKA0IblIzp19DQDQcCRnQqpSnUPFuTBGCVGH3v0pwSWq3bzi2rqQol1qrbaPrjOnTFJ+nCQ1sZt63csRfj0pOWeqRnM6VlNReawax/OiNi8uMeEdkamq2C8uihVWy+fnxRTRl891jEf2Ed9lQmpTjcR0xgwfQdtGijIhp9fWmU58oRJ1TScrGR3tzOR9JCa5liECc4jNevODW/yxGSbmKiVxZbxwiSJxe7LiMLB7z3YAwB07SVEe3S57vnGRoqIXqqoQS+SiGtUcLMtVY9bII3fLxQTt6NAkE9HCnLgQ7tq1EwAwvFUiRccmmSxn8jTTI268GR7vHdskyvjYBBWsqDeU2YbJdcN7rdqU58bV561XxWS1VKK9NT4nr7F6k+ZrsJf2YkWdDzY5tr2DfDpcbcdi844ryKKLe3BOmZoiSittzhQ3hhuV0L8C4An+/ASAL9/0SAICAgICbgprcVv8HIgAHTbGXALwOwB+F8AzxpiPArgA4EM3OgD3C+9cD4HkEnTLoTOTNeGCiEQyceXgroxLcYCHDlCOB8PEzPkLErRga9Tfti1533b/XUSofm9MylUtlUlaWpwh6T5Vk2v2MBkaKZLMZwnU3Bj/J58n8k+Tag0OGGnpX2s+3OzyC942VU5ytZ1t3ST5pHw67dfozDjoZLwiuxf2KpK4xOPtKcocWZZMMrGc19fPJB2LhZW6XPvlV0gjOrBbaJreA7t4PDqXBs19mSX6sqoM31/oDNbqFmLkhPC2ACqemmpZpOXaAgW/DOToYJ9yz6zFtLalSAJpGryQkSLkMiyh5/vpvDvf9BZ/7PgS7d3SvGiIS+MUaPXA+x/xbfccIk2ylwnqxTlV0u0ajbHckOyChs8zbcVI3DHntqg0Sv6bXElwZVH+GudMAoD4IGke2wryfP3co+8GAGQLMm//9wq5E8bs1jpzVNwLC4O0T149LW6t872s9RSUdsmLVWPiWGu2Me/xu/bf69sGJimzY7Mlz9csFw1Jp2nvRmq/OolbJ7xxLsVaMyyVaG2bLKnbSPqoOAldBYhVuxRdWSvW4uXy8ysceu9NXz0gICAg4A1DiBQNCAgI2CRY91wujihIItw0iYBl+Uk00RWzuaap1L8rE+wDrXzCy1Xysjmwl0wpka4C36DrDw+odLs5UuMWl2Z9W4kj2DJpUpFzuoiEG6oiS3w6Ta2esWpVZJOLPt9FxNaUycDda60ixM9y6FwxniSOOud0rSYXZwLT8+w+tVl32GE45qjQweFhf2yR/f21y3fdpeAtiq957FRd61IYK7NNmdTWSktVkGeVN6dub2KK8nycPk++2KOjMo6hXvZtVvupG+nnfLJ14Qo3bfPzEuF64dhzAICBNO2Ju/bv9MfuuOdN9L1BuZdymQnBlqjZqZjWtI8jjqPdMu65y5RC9tQrktY14liKihrH2GWaiAqn850Yk1iK8XH6bAYklsIVI2kjldGOJN/6RJNLFzz8K//Yf55qEPGYmhPz0Xe/9v8AAC21r8eGaB6ml/j8PjGXReyH/rp6HrPI83jF5OJmvMLPUE357Odi6i+fExPUHbvo88yCmIhctHedn9Vjr7zkj23dRut89qysi/N5v/c+MZn93ff+FgCw78CdNH71urX8XiotytjqXYj6tSJI6AEBAQGbBLeNhK7hSFF9LMul2dL8K9bQVeMd4ZdWxAWLYNW6SLqTHAX60OH7AAC7tgnR1tdHbo6T40KUXmNJ5+o1IVZdKbIt/eRqNbJNx1TRmDS54crX1RX5UWcJc4lL3JWXRGrx0rjqw81HubyyhN4eFUro6lKmjndzR9R9uLHpXi27cNW5tZIWyWeO28oLso5OaTAq2rTSIqnJuUimlDtdkYsfGOXqV2dtytZFZDx58ixdc4Eku7sPSsZByZOoMyV23qs/5AkwpZ3wftqyVcrBLQzS56kzzwMATpdkn2wbIclxuFdJwUyyXr1ySfpYJGkzxRG0S2XJwzLDJdqQESm1XKG2s68JWbht+G0AgK07SQrPRKL9HH2d9n+6KJJ/nTMvGit7sttW8fOX4KLbDd/93nP+8/QFil7NLsgeXrhKuWcyShOaHGdtld353vHmh/2x977txwAA33he+j01SZGczapye+a/TkIvLYmrZDbPGr7WGlljcvljAHhmfG6OpPZP/dc/8IcO3U3a17kTr/m2uzii+eyZM77tm9/8KgDgX/7OvwEAjF2RLKKzFbrm5bNnfVuDM2OqvJjXjSChBwQEBGwShBd6QEBAwCbBuptcnDlB+6G75FZtZgeX/MalZ1U+tBF/VmUCfQrU3l7xA96ylXxhe4ukDh++/35/rJdNKN/+thAuU+OkblVrQlo68qW2SKrv+EVJhLRUJ3Wyrvxek6JBnRnDpULQx5IiZ5PaOpHgN9xW4IKj1hJSdK41ba6D9uE2Ma3V+CSRkufHhKybKjOx2tLEcWvZaIEGMnye8yHXUY303VpDxnP5Cq3L0pwU05ifp7kcGaHIvoFelRiqlUASd5nKKEXjaScNXY1aMSntOUimjso03XNlScx1p4+Ten38hMxHBWQ6qaqETI6ATbMPe5yRqMnCCCnf1cZ53zZ54QSNsa3AC503PUP9jk0tqmO017NZMcO4CNco0umml5H3+t7dvmvpPSae6yvh1Df+xn8ezFJytVyvJMQbS9M4lpT9I+alb3KKXF1LNs2mp2JafNkb5Qr3r8h4F8/AkcpLyuTS30PvA10Mp8rPbW9O+nWPyXyF/PhRkT159AdHqK+8GEe+/yKt0ZGXXpQ+2Hx25TyZVZpq3a+xaeb5v/uab7tyha71oQ99BDeKIKEHBAQEbBKsu4TucmpoachJtW3pTn3uFC5BF0suC017OfQNUFTZ/rukDNa991AKzB7O9dCrotZ6MiTx9PVKIvuJMZLWl1RC/RRLby7i0f26A0CFyRUdxZroH+cLKLCbnpa8+T51tKkrnNHsmi62japc8bykNLvdjiWlmU2pMn0Ts0QkHXuNJMfxaxJJGWWINIzUWjW5kIh2IWxx3gzLEXV1JWdUmPicnhEp69oSSeZN5e7Ww8VQRoZJC0u1+dhx/hiVV6Vr+lwvoese3PnSb26EcopsvYtI9qvHRWO4ME7zsH2v5B3ZtosLPqhCCrGTNlO0J+OMaBYRa6O9/RIZOT1NxOu8Ik/PcX6Xb50kbVGnSH70PVTKoBQrd0hXzEVFP7oIxyQXVviiIVq7W50W3Tu43X+usva1WFH7g69R14+Lz0HNe15pu5VF2mux0phbvB7vedPbfNvxS1RM48w4Sc2a7Gxwzhft5ljn44sqX9D4OOV1GR+jwiNPfvSf+mMLFdqLVr0+F5f4vZSS8Ta5NOCfff4LrsUfc0Vt9h444Nv279uLm0WQ0AMCAgI2CcILPSAgIGCTYN1NLtarXToVZWd9z1TsfKZdi/wW1cukzuVjMQVs4SjMd7/lQd/27nc+CgDYNkjETL8iTF06zZ3Kr3xqgvzQBwbkvHPOd5cJlx41g5kinWdKonaV6hwtGStS1JlhOCEYMqL+FVgt66/K/S006fhMl9VKSn3bjk5VWtRrbtCRsy10nO8sPmdev+Dbjp4g9fbiGJGA84ty770DpFY2IkVi8di0wp6KXPIijpZUPuqOuFtUxFadTWAZI+vtfMeLXEnetnRUKJ3XnqtsZZOBL36jzVgula4yoUUc0br7XiLXXUInABi7QuaPvTvu8m39uw8CAKotnRSLTVtw8RV6fejvwPaDvm3rtnvoQ0X8lyOQ+SXNpN52lV72zv303dcnZT5KbIFoqD1jnS+4zxzcaXJpb1o9bPTy3KRck1PYjsSyFzxXruvn8jPR4CjtWFUi6uW0yrVY9qmLNxmbkWvNz9I61PgZnVJVyAq8pq+9+qpvu3qN9u6JUyd826nT5Od/331kpj2wT+a0WaV+rTLr9UREzmpTXppNu4ffTH7rE+OSnrdcpvfClpykio6vu2prJ4KEHhAQELBJsO4SuvtR0lF8Ef8qZ7MquT1LsS4vQ62qSS0mUGoihTzALonveOgh37ad84wMsotiLq2IVeaiDt1/n29yqW+/9e3vynks+WW4kENBSTll/nUeTos7U3ELXcvm5bczXaTrLkwTCdNUFcv7+Ed/pCRtY/MkERwviwTYASXZRUz+6Tl1Elhk9HlMwHIOEJ0Lx3LOi7l5Id9+wKlsX375mG9r8hYyTOY11LK4VLZpVUcyxXMTp7Sky0Spc+tqaumajhVyioBiMrmi3BuHC+ROmGcJPTaaaeuUW3SN1+WInTZlk+QdnTqY67TmiYi984F3+2MjB2ht+7aKxlc3TLYq0tK5/7nV1uN26YQzsRD1ew5Q3cupC6KxuDDPBt+nUaTrhWtUJKME0TKRdZrNyrlcNIzPDaQa1yCh17XLIT/TdSXVtlwRFfXcNq6Qm2e8lyTX0+OiDX7m688AAC5eFTdOl3Plr4+/4Nsi1pDrrO1+6Zln/LHd2ykPy0vfP+LbJqbJDXbfHokunpmhtiK7k85PiHTtXKhTKl1ylTX8SkXu2cTOuYPWfesWKV5iIro/l7sJAFrN1ed0NQQJPSAgIGCTYC0FLnYD+BMA20ExJU9Za//QGDME4AsA9gI4B+DnrLUzK/Wzhuv4z84OValIHhZJheKyLnYWLdgxKnk2Hn0HFQAYGBDpxhXAcPa/ckmkz1TsykqJhJRiCb6pbHwxy1IZHu9Aj0hDWc4l0zMttVP72f1qyw4Z2wxnzMtmqN9qWl2zSX1klc19ew99LmW6LVenbbz9MF9D8RLOTmo5h4SuRn/qPLnCvfyDH/q2adYoZhdV8Yg+koy3DFMekREVeIMUSTD1hvTrCnjAiFSWSnPQE7uCNsoitWzpp4CUYo/IHuWx0rI7Blpc1MDZ0p3LHwDYBLfWbkjFLtNkkmtjZ3CNy/MyMCQS2ABLY1oTsDy/tj0bDvfVWaowArtzKk5hcBe74UbiYteKyRWwkaW2zPAd/thUnbXQnqIaN7WlVNZHf19JW0cM63LNLm6fDo22MpF0Ly2VSTPLrsLNQRXQUyIJN2KtdD5SwWPTZCfXpfPS3F+jqsrdsfa+f5js3llVAvHrz/4lXUeXx+N7nxkXW/tAD41p+ho9B8e+L+vSVyQOrlAUrSfLmmemJu+UdJWe86kB0tJsU6T3Br+DUmovuP2WVtze9WItEnoDwK9ba+8F8AiAjxlj7gPwcQDPWmsPAniW/x8QEBAQsE5Y9YVurb1qrX2JPy8AeBXATgCPA3iaT3sawM/8qAYZEBAQELA6rosUNcbsBfAggOcBbLPWXgXopW+MGeny1RXhVJ8kk0tbm4ugdDVI021lFgAAo8rkMjJCKq9253P5YlyumHpTIg1LnMK23hRzQoaLNhR7xYzg+nO9FrJCgA4NkFprZqSPPjYj9FZEvc320necyltRrlmzs9RWrCoikc1A88oE1Q12WTEQAIicq5eatha781W5huep85KX5tvPU06K8TEhgxyps2OXFHLo53qQw9uI5IkVIZdhNT+tcm9U+R6qNbkXV8uxwaRoeUm2ZSZDY5udE2ueZdI0o/KZzHPxiLEpIrP6don5Q1ww5d67pRZ2+6SlXGllLypSb1k6k8SSm201C5i0bDOPeR9J16sc4zZt3IiLNKe79t/t25pLtEZ399M8920Rcq/FqXIbVhwAImfm0eYVT3x2jtHVF221Os163dw/c/1i5inupNfDwJC46W3ZSma6pT5Z79pFInGx+VCvAAAcKklEQVSXLlCE5uSlq/5YzNev15W5rsgmNlWO8+F7qcjE7iF6H4xPSB/9XFglnZLntsIpjI0yre4fHQUAbOV6ty7CmU6kP42GzslDZsBh9YzmuZjHVTaFpXR0KkfA6uTh8SrprteCNfdgjCkC+DMAv2atnV/tfPW9J40xR4wxR1wyqoCAgICANx5rktCNMWnQy/yz1tovcfOYMWYHS+c7AIwnfdda+xSApwBgdHS0Q4hZS5EFQAoiREzqaenJSVQ7dkiZrb4+DjBR8o27liNHsyoQqZ5yUrtIMhOz9LtVUjlD8kWS1lNMcCwuyY/UQo0+74jlXoaZlC0qMmh6kiSCQY6sWFJEbLpK5422hPR1uU4q6ZXJkrZScT4QpFMSbKjzUkzoVhZoLk+elwT8cYqudWC/ZP/bv4dIpnpFXObGmTS6dP409a9cr1JpuofBAcmw51CrikjlcrhkczQeVzwEAAzPZaUs0k2eA5CscndbYin/zHlyc9u5VZHhTL4lzVESnBYWKQLZtenvNS2Tp6ad2NRoC2ZKSFDoCFvhQuVgw5GuihS17OZ2/pQUV1i4TJn7tr2J3CZtRjSieovuXQetxExQoq1E4fKcSjq/ELsFryZBLotn2/uYFKfIchbMtJKkDbvEzhw9LfcyQyRohvfrFhXUN32cslk2lYQ+0KI987Y3v9W3/fh73gcAOHmGgoOefeaz/tjodpK8+wZEgzt/ioj/YUVyDvbRZx/kpt8VzoLQkJuJOCHNNTUHcYH3HeeKUVy/z+USKe282UoKCLw+rCqhG1rhPwbwqrX299WhrwB4gj8/AeDLNz2agICAgIAbxlok9HcC+AUArxhjXua2fwHgdwE8Y4z5KIALAD70oxliQEBAQMBasOoL3Vr7N1jZgfe9NzsAp+LFbXklOnVT55/rq9ErpijmqM1du4Wsy/joy07V2vWfUqSau5KuZ7nEfuoXLl/xbQ1WO4dYdVsYk6IGTVaHt6pE+VUm9YbzElW2nVW2kRZHoalk+03Q5z5Vg7TEal9PWkimbvDFLLQK5yJFc0JaFgcocjbuY6JZ+dRXy+STu3urkFjDfTTeJYiZqe/A3rZrtlSUZ4PvS+dVcUSScg3GIhcLmbhEfsZ9BZmrnTv30L2oCEqXOrmgig6cXCTSdIqLTVxSdWDv3kd+2c02v+iVTS5prl+r08VGbAJoM6EkFOsQJBWAYPNiW7pkV7iF10xFp0beN1wR+/y5viQ0VoXrXrp51ulz0zxuzc2mXaoklROlwSSdI5AjRRq6PDOmrRcevr6XZRaDtDKBxuwUUDohxPuFE6/ytdXDzDEO81PkWz+4S/rYOULP3HC/pOW9/xClJL7/sJhcpqeJWJ0Yoz1QKMi+vvseiiA/cuR537ZlgEx824fENFhg8tk5RugCPEl5kVwEdF3t/xabBlMuNbGiQF2907gl/UYrb8k1I0SKBgQEBGwSrH8uF0ZSibY2CZ2FA1dcoal+1XOc82XbNkni75y99K/o8qINTe2Cxm26fNzp05TRzuV6AIC5Ckd5DlNi+j13iKvk0aMvAQDKKSFWp3iGR1Xeh/4lluxqdP1sXkW+xSSt9qlIxxprEpdmxHVPHNPa7w0QSVQXycgViOTJqWhGJww2yiQN9aREgtjaz3M6JFrBYJHa9u+SDIIpr40wgackFBmRtNWZyGw2ZWxljg48foJIrKtXz/lj/YM07q3K3a3BFd6zLdFsegvU3+wctb1+UQje3TtIosupSNtuLmLOda9N+kwg7VO+XBv9aSm10WUMtdr9j7NDmqaKUDb0ueZIeStaY44l4paSjDOspQ3t2OPbGlXqo2/HnQCAKC2EepoLOuh7aVnaY1nlxxCzu641JM3WY5GCm4b3bnP16FCNvq9KObaLp84BAHpUxHTk3GvVXpg7TetW55KCdaWp7nsHkb57hvb6tsIAOSmcOX3Stx0/+gMAwA+Pf5+GrTSzySmS2hsLouEMsZtnRs2by6Tp3jf6+RLSvDPDqa5n02q6XEks0avz06wm1dqKtIhWdKMIEnpAQEDAJkF4oQcEBARsEtw2JhcN8f/tjExzaoxWIfv62ZzQIyqTpPrsTNfZaLgalnJNV+NyWpk1jh2jNLHzi1KTMGLzzmSJzBTvf98H/bEd+0h1e/k7f+vbahy911JjS4HUrFKD1MqyERX8LfeR2WgopUhiJk5a57AmOPNRKiOmnxL7js9dkhS8zhe8yWaSe3aJ7/Zb7yaCub9XTC5uXdJp2TYtJjmtN2MJxKwh4zC+mIcyB+Vobh54kKIf4+NSQ/P8WSLO9h+837elh8n8crkmazW1SHPajEgFH18UVfbiOJFkd+2RIgXGruzzGzHJjqbef8tHrcx5vvaFko9Y9261pTBmE1F92rfFoHXJMwmZzsoatJq0dxpQ+5r7jRRBbjJE6jW5bmgjEvNenKFr9uQkgjHHBTGKKqFVk80IjRSZXGaa+l7YxLDGdLsO20+JM8Hka+cAACeNjpimdayrwg8NJkVdkrXWjBDw165Qf+85/A+k7RLFHbz4kpCcx4+RM145IbK6zrVYD+7aJePg5ySVFXNXnOZkfUw0W2X2cs9XTaUHjhPMdC6XmYv9MG2J2th8pXjm6TnaFwOQOqPXiyChBwQEBGwSrLuE7jysUiklybhfMsUwxC5SlKWFVizSxWA/SRX5nCpYwX3UVQL5ZoOJH3bXqqhcDK4Yw2sqn8lLrxBJFylpLucqpfM4apEQGT/22GMAgN37xX3y1aNEDM0vqV/nIufXiGlskzVJ22kXqf+hreKutZAmievgW0XCVAF3dLvKRdGRNk2V4rTuqp2rn/A8SyQxpyIeHhTp0EU9NpJc/ZQU4lfNuZ8mkI3txPTyiESReFz5uEP3SZGRo8eI7DqjIiP33UXHMwPKPXSSpPWWoXtpKlnlxGU6tn27EH296ZVlGedBm9KFKLwEpsv0re5n1pZxyNL+S2dl9XJNrmQP/qsKeUyU6F40sZriyNmREdkf2RqR9iN5ylmSLkikratyn7FyzVEmEjNNkfxnS7TOcw2a06ihNErn8nqdJdImlGbb4rmMa7Kf6rMkLS9NidbY5Dl1xVfq6hmNeO6/97fPyTXG6Xkdm5ASdLksPS+7RkkK37Nz1B/bfwe5sGaUljk9TkRpuSQR0K5kXoGdCbQraFJUu0jmnY4W/nlsKw3JqavV+em8en/dIIKEHhAQELBJEF7oAQEBAZsE625yMb5ii8CrIVqjXVbXMFIHFxYooc/CvPiWulSsOVXpxqnLltWpSlnIyPkyqabHXpXK3+cvkjrXo8jFrKuMw4yH0iBRYx/2ew8LgbdnL6n5V46e8m25XVztiNPtZgel/4FtXA1F1R9scn3Ppbrc8/957gfQMAl1M7VqmM1QH47s0XCqobYguHVJSjOrzTAuum215GorXbNtvLwLCspn/9C9RJQee03m7/w5WqOtO/f7tt27aJ7HJmgv1K34vs8yOXZ1UqrfFLaLeWk5XI3TthhPXnerIzmj1e+5fQuTiSutogOrs0SEFbjmbEaZDfMxmeZqOg2yS7lcE2K1P0sE/R0FipuIihKlcG2G1r00I/MxtUTml1i5Pc/UaF+UXDRrRkWnwvlTC2m4Flx9h6T4PXOErl9rqgxV8+Rznm3I/X3kF38JAHDsFdrfL74g9XwnLpJpZCASQvjQYUqVW3peHBHKHDWcZlNpWplMyyUyA9X02vF7IZdVSc3YSaLM7wgdKWoTaqw6It2q5HTW+aY7B4e2ClhcoUrL1Nf5DCUhSOgBAQEBmwTrL6EnJMp3BKiOvHMEqUuHq0mKGXY1vHxZEtnfx8SaVVKIK4jQ4LSXWtJ0EWlHX37Zty3OE1nT16OKAyzzqGyrQcrkh45u2865SB66T3JNFGo0jjL/ms/WhTyaW6JrXl5Y8G1TM0SaVipri9RzZF2s0vi6+XUum+oWYCJXz7Ib2SN9JKWhjRLctpLOT5LMfR6fBJKxmKcFPHTvnb7tB+zWeOmsEKXbth0EAOzcStJbI5LrzC/ROOamhThrDIs0thxpjihtNTXR3OlKKwUfVpastNtig9PtGpUG2XH2i3O03rWW7Os615pN5VX6YU7b3J+TPXNwF+eqSdG9n5+Say7E5Bpot8gennO1LdW6VLN8nN0nI0WiOgm3pTQFT9R24YUvn5U8R7U5cj/s6RHtKzNExO6bDt/r2976Vkq5G3Ok5rGjoon2gJ6bO/cf9G3PfuNrAIDJcXn2d7Bbq88JpKKXZ5gAzWRE2+hhrSidESI4x+Ns8Hf1s2HZAaGl5OE6q+o6J5Uj0t0zZ9Q7K6mwj1Va5Y0iSOgBAQEBmwTrLqE721R7NjNCu5uPg5M+5Xzn/D85KTlXSmwTT6UlwMRJok52S6ngnRqLSleuiFSRZqksq3KAuACaFGsR9Ubnr2ocya//LOekQEOkQ8MuYlNLZNtdVNXXm5xTo6Ek/ypLriZe2a0pSfLVbe7ek3Lb+JIGCRkIE7MSJjV1yV6YhKRxuFR/NsGW35OTOb3/LpLW4+Nnfdvs+HkAQL6PXFhjlUlwiO3B2wbE9prqYq505y2o4iUtNphGqtBBg/eMcw9tz9vibKTqQk5LikQ7cFk74zIF16QHpJJjqUbnzap9HRfomoPDigthye7CNZqrxhbJexNnSbpv6PJxGdIGIqUNpA1J5k2fIVC5SvL+1ApzkgvrcowayXKYP0BaRLUmz+Mi8x2DQ5KD6Tt//S0AkpulUBAVe/waPZt/+qdSsKLFNvkRleunwG6LKX5H6KVOcUEJl6MFkKChltZYOBuicwd2rr30XWcT15oq/dVu0u5d4aZNS/nVSm15F/49djMIEnpAQEDAJkF4oQcEBARsEqxqcjHG5AA8ByDL53/RWvs7xph9AD4PYAjASwB+wVq7PIBxzUgi2jTBIHoTk6Ip/VtE583PCZG4yFFqug+XeyFyNQGVCjQ5SSaR6UkxjfRy/pWsupYLXIs50ixOdeZ4MKogQYUJ0qmGRKG1eBwNR6xabRZiEkalB846cqW58vRaZV5x6n6sSU6eXh3J6avKO116lXSxbi41WejO83krdGRfQr3YblF2zl3V6Fqebi6Vuj/US5GOD9x3j2+7dJVc4KpNmsueoqjqhV6KnOzR0aGplWWZrRwxq90SF0pVHoYifXntHf+qCVDnRtrmjus5VCEGC/1kKsjneG17hQCNK0SeZpQmvlC9BAAoKYJ3vkwnnD5BuYcG75TI4+373w5AaosCQLlC303nlWmGU8fWXX1U5Z4ZG9rrUaTJbV6zLpa2q6oQyuI1MhvpeqApzheTUZP00isv8QVcx+r5ZfNHWm2/gT5aq3xG1jvPbq8R71dtXoGPgNYOF2xWUWbfNDtQVDjfjXbfrfPzq02aUULNWRfx3uTnq6FT6/Ix7aTQ6pJfaK1Yi4ReBfCYtfYwgAcAfMAY8wiA3wPwB9bagwBmAHz0pkcTEBAQEHDDWEsJOgvAiZdp/mcBPAbgH3H70wD+NYBPXu8A3C/aaqRe7MkMGrJ2O4oiIgvHxqTs2MULJMk0RiV/R7XCGQTZnamqCIxLlymIqLIkknSeibi0cjdy10+n6ZptwTgs0fX3iZS1rYeJOKOlGyeR86+zlXE4kqehKtq3GnRNXRTi1XPipgUkuwumNHHMxJmWrpe7FRodROS80nSV+0SSmuCl8ThBRkgYW7u7Fkt7XoLtzOtjlOxhXO6XrJDErsxczXa6g0UJeX28epSAutc6OgusaEbYGlcOkSVBxRoaf54q3MLnt2IhRU2T9kdsiUCsqMCUCrvXLiiXvOo8kb+5jATFFVm6PryVy9i1JB9R6wpJvCYle7JSoQC85haRfnsyvW6Q9AdaS4rcDcu4TWfW0+VYuCxZFN3UtEnBLLJeuyL3Vy4TGdnXR1qYy0FEF+WxZpV7ITssFLU7JGvPLiBQ70lfuELt9TQ/J1Zpl674jd+bauu7RJS6YIV7VzXanCRcTiUnjcv+Mz4bpyoD2OX5WivWZEM3xsRcIHocwF8BOANg1orj5CUAO1f47pPGmCPGmCOlUinplICAgICANwBreqFba5vW2gcA7ALwMIB7k05b4btPWWsfstY+lM+vHMwREBAQEHBzuC4/dGvtrDHm2wAeATBgjEmxlL4LwJWuX165TwDLfKYT1HKnoriIPe1DnmHVakql4bx6hUiyHcrkMjdHfq8uTezCkkTbnXudfJr7iuI725sjokUTji4tq8uN0pbXhImR0e2SrnNpiVTGSkW0k3S2vS5qNqMKEqToc00liVlaJFWtLQ/GMrSZXFxbwm+snmdnPvJkpyZ0Eq7hv6vXhf82EvzxvdFB9ZtYCMCZfLoQpvpYnQtzoKqKJeSZCGP1tqXMK2aB1iCtImebXUjR8RnaR426kNBOQ9f+6y6fUIr/phVpWOToYk2oVzgXz2RZxRg0mcxjk0vDiNBT5nTPExdf921bIoqK7h8WEjDLavtdOzhKVuUveu0KFYBo5pXJj/9qU6JLV+vuMzL6mHtGtekiYR3RDmfeAIAWR1w2y2JCqXLk8/y8PLfDw+ST7pa7qopUFLJ0zwUlGBq+xqx6lh2PnmczjG0rLGHdTclAnYlSkeYljtR2zg8tVRfXFapoC2RH0ny0/CegPcW1zJZcM8nsfL1YVUI3xmw1xgzw5x4A7wPwKoBvAfhZPu0JAF++6dEEBAQEBNww1iKh7wDwtCFfvAjAM9baPzfGHAfweWPMvwXwfQB/fCMD8KSD+mVzv/7aVchJ5CnOFqhJ0VzOSSvyqzsyQpL54qKQnMdfpRwgB/ZRfhVNej366CPUbyyEy8svvACgnUi0HCmY5mvqX92Gc1NSEsH0hMuKJ7++IwUiqEpMCtWUP5OLJquoH/OzTNgmCMEebdJ1ArkiJIy61rLITKt9A7vkZtFwR2WOOsu2aebYNylpxH32662LX7hcMeqaKXYpi5RrnbuC0zrqyi3NZrlNEdOVaLk8KWgmELwuSWWxRyTjHEuMhqXPDEQrGOaiK32qYMU8l1drTsh5C67MXA9FiDayoiGaMpGFxYyMo8js3MCgSKl9eZc5kvovN0UbbDIRqyNQx1l7LY1d8217B7bRB75no/KK+GVB53y3aU5oh61IS40lbU1GNqpc9nFWZY4coHuZmaE2ve59edJAXCQoDwBA+75274OkYituA+r91+RnOKVcJJ17pc89ozNeuvJ4CXu4Pbtmu8Sth9NouD3fWfzlZrAWL5ejAB5MaD8LsqcHBAQEBNwGCJGiAQEBAZsE656cq6UoPIEjLpSqzuqI5QryDeWTXWN1v6B8UfsH+7h/IVVa3EeFkw1lVA2/Pfuo/uD2nVK3s8iJnr759Wd9W90l1cmRH3C5rBI4WVaZVHIup74vqvqKPWXy+TWs4mlDhkuR27TSR6XGNVArawvETfINdj7maaXiecLR+dzq893fBDNLkgd3UqpcHy2pvtDNl92n4m1LpUx96PqoXk3Nyvats9rs/Yf1MV6WmjKPVboQzHEq29HmzQ6K6EvzfsoxcZZTibtcHdqSWvcc74tt/WIucSalBpOjVvWRYfNRMSfz4XLN1VScQrM/4mvR/tC1cku8/YsFiQrddRft9blrErfhIhczzszUEvXfmbb0vSdh+YxW5sTcmXYmUuVn7+ICJqckstXtj6vXyMxYVMVO+npo3nqU2cuRmzlVhCbvPjtzjE7fzL7gbSS+27tVeb687zib7lTwNwybS5pN3Qf9aXNE8Ft2ZeeE68xptyqChB4QEBCwSWCuN+3pzWB0dNQ++eSTt+x6AQEBAZsBn/jEJ1601j602nlBQg8ICAjYJAgv9ICAgIBNgvBCDwgICNgkCC/0gICAgE2CW0qKGmMmACwBmFzt3Nscw9jY97DRxw9s/HvY6OMHNv49bKTx32Gt3braSbf0hQ4Axpgja2Frb2ds9HvY6OMHNv49bPTxAxv/Hjb6+JMQTC4BAQEBmwThhR4QEBCwSbAeL/Sn1uGabzQ2+j1s9PEDG/8eNvr4gY1/Dxt9/B245Tb0gICAgIAfDYLJJSAgIGCT4Ja+0I0xHzDGnDDGnDbGfPxWXvtGYIzZbYz5ljHmVWPMD40xv8rtQ8aYvzLGnOK/g6v1tZ7gIt/fN8b8Of9/nzHmeR7/F4wxmdX6WE8YYwaMMV80xrzGa/HoBlyDf8576Jgx5nPGmNztvA7GmE8bY8aNMcdUW+KcG8J/5Of6qDHmLes3csEK9/DveB8dNcb8L1eNjY/9Ft/DCWPMT6zPqG8Ot+yFzhWP/gjABwHcB+DnjTH33arr3yAaAH7dWnsvqI7qx3jMHwfwrLX2IIBn+f+3M34VVDbQ4fcA/AGPfwbAR9dlVGvHHwL4mrX2HgCHQfeyYdbAGLMTwK8AeMhaewhADODDuL3X4TMAPrCsbaU5/yCAg/zvSQCfvEVjXA2fQec9/BWAQ9baNwM4CeC3AICf6w8DuJ+/85/5nbWhcCsl9IcBnLbWnrXW1gB8HsDjt/D61w1r7VVr7Uv8eQH0ItkJGvfTfNrTAH5mfUa4OowxuwD8QwCf4v8bAI8B+CKfcruPvw/Au8ElDq21NWvtLDbQGjBSAHqMMSkAeQBXcRuvg7X2OQDTy5pXmvPHAfyJJXwXVEB+B9YZSfdgrf06F7YHgO+CCtwDdA+ft9ZWrbWvAziNDViR7Va+0HcCuKj+f4nbNgSMMXtBpfieB7DNWnsVoJc+gJGVv7nu+A8AfgOSbn8LgFm1qW/3ddgPYALAf2ez0aeMMQVsoDWw1l4G8O8BXAC9yOcAvIiNtQ7AynO+UZ/tXwTwVf68Ue+hDbfyhZ5U6GZDuNgYY4oA/gzAr1lr59d7PGuFMeanAIxba1/UzQmn3s7rkALwFgCftNY+CEodcduaV5LAtubHAewDMAqgADJTLMftvA7dsNH2FIwxvw0yqX7WNSWcdlvfQxJu5Qv9EoDd6v+7AFy5hde/IRhj0qCX+WettV/i5jGnUvLf8fUa3yp4J4CfNsacA5m4HgNJ7AOs+gO3/zpcAnDJWvs8//+LoBf8RlkDAHgfgNettRPW2jqALwF4BzbWOgArz/mGeraNMU8A+CkAH7Hit72h7mEl3MoX+gsADjKznwEREF+5hde/brC9+Y8BvGqt/X116CsAnuDPTwD48q0e21pgrf0ta+0ua+1e0Hx/01r7EQDfAvCzfNptO34AsNZeA3DRGHM3N70XwHFskDVgXADwiDEmz3vK3cOGWQfGSnP+FQD/hL1dHgEw50wztxuMMR8A8JsAftpaW1KHvgLgw8aYrDFmH4jg/d56jPGmYK29Zf8A/CSIWT4D4Ldv5bVvcLw/BlK7jgJ4mf/9JMgO/SyAU/x3aL3HuoZ7eQ+AP+fP+0Gb9TSAPwWQXe/xrTL2BwAc4XX43wAGN9oaAPgEgNcAHAPwPwBkb+d1APA5kL2/DpJeP7rSnIPMFX/Ez/UrIG+e2/UeToNs5e55/i/q/N/mezgB4IPrPf4b+RciRQMCAgI2CUKkaEBAQMAmQXihBwQEBGwShBd6QEBAwCZBeKEHBAQEbBKEF3pAQEDAJkF4oQcEBARsEoQXekBAQMAmQXihBwQEBGwS/H+1TfXHifKg+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GroudTruth:  ['  cat', 'plane', 'plane', 'truck']\n"
     ]
    }
   ],
   "source": [
    "test_iter = iter(testloader)\n",
    "images, labels = test_iter.next()\n",
    "\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print('GroudTruth: ', ['%5s' % classes[labels[j]] for j in range(4)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted:  ['  cat', 'plane', 'plane', 'truck']\n"
     ]
    }
   ],
   "source": [
    "outputs = net(images)                   # net是模型实例，应用的话直接当作函数来使用！代替了net.predict\n",
    "_, predicted = torch.max(outputs, 1)    # outputs和outputs.data 都阔以~\n",
    "print('Predicted: ', ['%5s' % classes[labels[j]] for j in range(4)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 56%\n"
     ]
    }
   ],
   "source": [
    "correct, total = 0, 0\n",
    "with torch.no_grad():   # 应用时要设置no_grad???\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "        \n",
    "print('Accuracy of the network on the 10000 test images: %d%%' % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What are the classes that performed well, and the classes that didn't perform well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_correct = list(0. for i in range(10))\n",
    "class_total = list(0. for i in range(10))\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        c = (predicted == labels).squeeze()\n",
    "        for i in range(4):                      # 因为batch_size=4，所以每次labels只有4个\n",
    "            label = labels[i]\n",
    "            class_correct[label] += c[i].item() # Use torch.Tensor.item() to get a Python number from a tensor containing a single value\n",
    "            class_total[label] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of plane : 58%\n",
      "Accuracy of   car : 67%\n",
      "Accuracy of  bird : 50%\n",
      "Accuracy of   cat : 31%\n",
      "Accuracy of  deer : 50%\n",
      "Accuracy of   dog : 59%\n",
      "Accuracy of  frog : 60%\n",
      "Accuracy of horse : 62%\n",
      "Accuracy of  ship : 57%\n",
      "Accuracy of truck : 69%\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print('Accuracy of %5s : %2d%%' % (classes[i], 100 * class_correct[i] / class_total[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.6 Training on GPU\n",
    "\n",
    "First define our devices as the 1st visible cuda device if we have CUDA available:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
       "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
       "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.to(device)  # Recursively go over all modules and convert their parameters and buffers to CUDA tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remember that you will have to send the inputs and targets **at every step** to the GPU too:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs, labels = inputs.to(device), labels.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Data Parallelism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0')\n",
    "model.to(device)                 # put the model on a GPU\n",
    "mytensor = my_tensor.to(device)  # copy all tensors to the GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please note that just calling ***my_tensor.to(device)*** returns a new copy of *my_tensor* on GPU instead of rewriting *my_tensor*. You need to assign it to a new tensor and use that tensor on the GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
